Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::Arc;
22use std::sync::atomic::{AtomicBool, Ordering};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::agent::{self, AgentOutput, Invocation, SeatState};
30use crate::blind;
31use crate::config::{AgentSpec, Config, LeakPolicy, MergeMode, Prompts, ResolvedRoles};
32use crate::git;
33use crate::land;
34use crate::proc::Quiet as _;
35use crate::prompt::{self, CandidateView, Turn};
36use crate::run::{
37    BaseSync, Candidate, CommandOutcome, DeliberationRound, DeliberationTurn, FixRecord, Judgement,
38    MergeOutcome, QuotaLoss, ReviewRecord, ReviewRound, RunState, RunStatus, Tally, VoteRecord,
39    tail, write_artifact,
40};
41use crate::verdict::{self, FinalVote, FixReport, Position, Ranking, Review, Severity};
42
43/// How much verification output is kept and fed back to the fixer.
44const OUTPUT_TAIL: usize = 8_000;
45
46/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
47/// a base that moved before giving up and leaving the run `Blocked` for a
48/// person.
49///
50/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
51/// that keeps moving faster than a run can catch it is not something more
52/// rebasing fixes, it is a person's call. Not the same *number as*
53/// `land_rounds` - this budget is spent before a pull request exists, land's
54/// after - but bounded for the identical reason, so it uses the same
55/// default. Counted across both call sites in [`Runner::finish_after_tally`]
56/// (once before review, once before the gate), because either one finding
57/// the base still moving is the same signal.
58const BASE_SYNC_ROUNDS: usize = 4;
59
60/// One queued agent invocation.
61///
62/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
63/// CLI hung up on its own stream is asked again from the same job rather than
64/// rebuilt from scratch. See [`Runner::resume_undelivered`].
65#[derive(Clone)]
66struct SeatJob {
67    spec: AgentSpec,
68    seat: SeatState,
69    cwd: PathBuf,
70    prompt: String,
71    timeout: Duration,
72    allow_write: bool,
73    sessions: bool,
74    artifacts: PathBuf,
75    stem: String,
76}
77
78/// How the graph reads one agent invocation.
79///
80/// Quota is split out from an ordinary failure on purpose: a rate-limited call
81/// is known to fail again if retried now, so the retry loop must not spend an
82/// attempt on it. `Dropped` is split out for the opposite reason: unlike
83/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
84/// error JSON, never the agent's answer — a caller that matched only
85/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
86/// left to read that JSON as if it were usable output. `resume_undelivered`
87/// is the only caller that acts on it; everywhere else it is reported like an
88/// ordinary failure.
89enum AgentOutcome {
90    /// A usable output.
91    Ok(AgentOutput),
92    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
93    Quota(AgentOutput),
94    /// The CLI hung up on its own stream after billed work. See
95    /// [`agent::AgentOutput::work_undelivered`].
96    Dropped(AgentOutput),
97    /// Any other failure: a timeout, a bad exit code, an empty reply.
98    Failed(String),
99}
100
101/// A request to park the run at its next node boundary.
102///
103/// Cloning is how the request travels: the loop keeps one handle and hands a
104/// clone to each [`Runner`], and every clone points at the same flag. There
105/// is no channel because there is nothing to send - the only message is
106/// "park", it is idempotent, and a flag cannot be missed by a receiver that
107/// was not listening yet.
108///
109/// The boundary is what makes this cheap. Every node writes the run's state
110/// before the next one starts, and every node skips what is already recorded:
111/// `prep` returns early once candidates exist, `implement` asks only the seats
112/// with nothing on disk, `judge` returns early once judgements exist. So a
113/// parked run resumes into exactly the node it stopped before, and no agent
114/// work is thrown away. Killing the process mid-node, by contrast, loses
115/// whatever the seats in flight had not yet written - which for an implement
116/// wave is an hour of paid work.
117#[derive(Debug, Clone, Default)]
118pub struct Pause(Arc<AtomicBool>);
119
120impl Pause {
121    /// A pause nobody has asked for yet.
122    #[must_use]
123    pub fn new() -> Self {
124        Self::default()
125    }
126
127    /// Ask the run to park at its next node boundary. Idempotent.
128    pub fn park(&self) {
129        self.0.store(true, Ordering::SeqCst);
130    }
131
132    /// Has a park been asked for?
133    #[must_use]
134    pub fn parked(&self) -> bool {
135        self.0.load(Ordering::SeqCst)
136    }
137}
138
139/// Drives one run.
140pub struct Runner {
141    /// Run state; public so the CLI can report on it.
142    pub state: RunState,
143    roles: ResolvedRoles,
144    sem: Arc<Semaphore>,
145    /// Set when someone wants the run parked at its next node boundary.
146    pause: Pause,
147}
148
149/// The commit a run branches from: the base branch as the remote has it.
150///
151/// Two failures this replaces. A run used to branch off `HEAD` and so refused
152/// to start on a dirty tree, which made `magi serve` decline every task for as
153/// long as the operator had work in progress - most of the time. Branching off
154/// the *local* base branch fixed that and introduced a worse one: `land` merges
155/// the winner on GitHub, nothing updates the local ref, and the next run
156/// branches off a base missing everything the previous runs landed. Two tasks
157/// in a row from a phone would have had the second silently re-implementing
158/// against stale code and opening a pull request that reverted the first.
159///
160/// Only refs move here - no checkout, no local branch, no merge - so it is safe
161/// with uncommitted work in the tree. A machine with no network still starts:
162/// the fetch may fail and the local tip is used with a warning, because
163/// refusing to run offline is a worse failure than running against a base the
164/// operator can see for themselves.
165///
166/// One function, called by both entry points. Two answers to "where does a run
167/// branch from" is the kind of drift nobody notices until a diff is wrong.
168async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
169    let tracking = format!("{remote}/{base_branch}");
170    let fetched = git::fetch(repo, remote, base_branch).await;
171    if let Ok(out) = &fetched
172        && out.ok()
173        && git::rev_exists(repo, &tracking).await
174    {
175        return git::rev_parse(repo, &tracking).await;
176    }
177    let why = match &fetched {
178        Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
179        Ok(_) => format!("{remote} has no {base_branch}"),
180        Err(e) => e.to_string(),
181    };
182    tracing::warn!(
183        "could not read {tracking} ({why}); branching off the local \
184         {base_branch} instead, which may be behind"
185    );
186    git::rev_parse(repo, base_branch).await.with_context(|| {
187        format!(
188            "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
189             branch that exists"
190        )
191    })
192}
193
194impl Runner {
195    /// Start a fresh run against `repo`.
196    pub async fn start(repo: &Path, instruction: String, config: Config) -> Result<Self> {
197        let repo = git::toplevel(repo).await?;
198        let missing = agent::missing_programs(&config.agents);
199        if !missing.is_empty() {
200            bail!(
201                "these agent programs are not on PATH: {}. Fix the roster in \
202                 magi.toml or install them.",
203                missing.join(", ")
204            );
205        }
206        let base_branch = match config.merge.base.clone() {
207            Some(b) => b,
208            None => git::current_branch(&repo)
209                .await?
210                .context("HEAD is detached; set [merge] base in magi.toml")?,
211        };
212        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
213        // Still worth saying out loud. The operator's uncommitted work is not
214        // part of this run, and someone watching a candidate fail to use a
215        // change they just made deserves to know why.
216        if !git::is_clean(&repo).await? {
217            tracing::warn!(
218                "{} has uncommitted changes; they are not part of this run, \
219                 which branches off {base_branch} ({})",
220                repo.display(),
221                &base_commit[..base_commit.len().min(8)]
222            );
223        }
224        let roles = config.resolve_roles()?;
225        let max_parallel = config.graph.max_parallel.max(1);
226        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
227        state.event("start", format!("run {} created", state.id));
228        state.save()?;
229        Ok(Self {
230            state,
231            roles,
232            sem: Arc::new(Semaphore::new(max_parallel)),
233            pause: Pause::new(),
234        })
235    }
236
237    /// Open a review-only run against work that already exists on `branch`.
238    ///
239    /// The expensive half of the graph is the implement wave — measured at
240    /// 111 and 134 internal tool-loop turns on this repository, against a
241    /// handful for a judge or a reviewer. The cheap half is worth running on
242    /// hand-written work too, and there was no way to reach it.
243    ///
244    /// No new state and no schema change are needed: a run with **one** viable
245    /// candidate and a tally already decided degrades `execute` to exactly
246    /// review → gate → merge, because `judge` skips a single-candidate field,
247    /// `deliberate` has fewer than two first choices to reconcile, `vote`
248    /// returns early, `tally` is already present and `fold_losers` has no
249    /// losers. Resuming such a run therefore does the right thing as well.
250    pub async fn review(repo: &Path, branch: &str, config: Config) -> Result<Self> {
251        let repo = git::toplevel(repo).await?;
252        let missing = agent::missing_programs(&config.agents);
253        if !missing.is_empty() {
254            bail!(
255                "these agent programs are not on PATH: {}. Fix the roster in \
256                 magi.toml or install them.",
257                missing.join(", ")
258            );
259        }
260        if !git::branch_exists(&repo, branch).await? {
261            bail!("no branch `{branch}` in {}", repo.display());
262        }
263        let base_branch = match config.merge.base.clone() {
264            Some(b) => b,
265            None => git::current_branch(&repo)
266                .await?
267                .context("HEAD is detached; set [merge] base in magi.toml")?,
268        };
269        if base_branch == branch {
270            bail!("`{branch}` is the base branch; there is nothing to review against");
271        }
272        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
273
274        let roles = config.resolve_roles()?;
275        let max_parallel = config.graph.max_parallel.max(1);
276        // The commit subjects are the closest thing to a task statement that
277        // existing work carries, and the reviewers are told as much.
278        let log = git::log_oneline(&repo, &base_commit, branch)
279            .await
280            .unwrap_or_default();
281        let instruction = format!(
282            "Review the work already on branch `{branch}`. There is no task \
283             statement: what the change claims to do is whatever its commits \
284             say.\n\n{}",
285            if log.trim().is_empty() {
286                "(no commit messages)"
287            } else {
288                log.trim()
289            }
290        );
291        let mut state = RunState::new(
292            repo.clone(),
293            base_branch,
294            base_commit.clone(),
295            instruction,
296            config,
297        );
298
299        // An attached worktree, so the fixer's commits land on the branch under
300        // review rather than on a detached head nobody will look at again.
301        let worktree = state.worktree_root().join("under-review");
302        if let Some(parent) = worktree.parent() {
303            tokio::fs::create_dir_all(parent).await.ok();
304        }
305        let path = worktree.to_string_lossy().to_string();
306        git::git(&repo, &["worktree", "add", &path, branch])
307            .await
308            .with_context(|| {
309                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
310            })?;
311
312        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
313            .await
314            .unwrap_or(0);
315        if commits == 0 {
316            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
317        }
318        let files = git::changed_files(&worktree, &base_commit, "HEAD")
319            .await
320            .map(|f| f.len())
321            .unwrap_or(0);
322        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
323            .await
324            .unwrap_or_default();
325
326        state.candidates.push(Candidate {
327            index: 0,
328            label: 'A',
329            // Not an agent id on purpose: nothing in the roster wrote this, and
330            // the stats tables must not credit anyone with a win for it.
331            agent: "(existing branch)".to_owned(),
332            branch: branch.to_owned(),
333            worktree,
334            summary: String::new(),
335            stat,
336            files,
337            commits,
338            empty: false,
339            failed: None,
340            duration_ms: 0,
341            folded: false,
342        });
343        state.tally = Some(Tally {
344            first_choice: BTreeMap::from([('A', 0)]),
345            borda: BTreeMap::new(),
346            winner: 'A',
347            rankings: 0,
348            unanimous_initial: false,
349            deliberated: false,
350            changed_votes: 0,
351            unanimous_final: false,
352            tie_break: None,
353            // No panel sat, so no quorum applies. Zero judges is the correct
354            // number for work that never competed, and must not be reported as
355            // a collapsed panel.
356            judges: 0,
357            present: 0,
358            quorum: 0,
359            met_quorum: true,
360            uncontested: Some("review-only run: nothing competed".to_owned()),
361        });
362        state.status = RunStatus::Reviewing;
363        state.event(
364            "start",
365            format!(
366                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
367                state.id
368            ),
369        );
370        state.save()?;
371        Ok(Self {
372            state,
373            roles,
374            sem: Arc::new(Semaphore::new(max_parallel)),
375            pause: Pause::new(),
376        })
377    }
378
379    /// Reopen an existing run.
380    pub fn resume(id: &str) -> Result<Self> {
381        let state = RunState::load(id)?;
382        let roles = state.config.resolve_roles()?;
383        let max_parallel = state.config.graph.max_parallel.max(1);
384        Ok(Self {
385            state,
386            roles,
387            sem: Arc::new(Semaphore::new(max_parallel)),
388            pause: Pause::new(),
389        })
390    }
391
392    /// Walk the graph to a terminal state, skipping nodes already recorded.
393    pub async fn execute(&mut self) -> Result<()> {
394        // Moving again, so it is no longer parked. Set before the walk rather
395        // than in `resume`, so every way of re-entering the graph clears it
396        // and a card cannot claim a run is waiting to be resumed while the
397        // agents are already working.
398        self.state.parked = false;
399        // Any seat this state still lists as answering belongs to whatever
400        // process last drove this run — this one included, if it crashed
401        // mid-wave. Cleared and flushed immediately, before anything else
402        // runs, so a resume can never show a seat as live when nothing is
403        // asking it anything yet; the node that actually dispatches the next
404        // wave repopulates it.
405        if self.state.clear_active() {
406            self.state.save()?;
407        }
408        // A run that already lost its quorum never resumes into the verdict
409        // machinery: `deliberate` and `vote` would otherwise clobber the
410        // stalled marker back to Voting and the run would keep going past a
411        // verdict that is no longer trustworthy. Everything already recorded is
412        // kept, so the run stays resumable (or foldable) for a human to pick up.
413        //
414        // On --resume the run gets one chance to repair itself: the seats a
415        // rate limit took out are re-asked. If their quota has since reset and
416        // the quorum is restored, the run picks up and finishes; otherwise it
417        // stays stale and still-resumable for a later retry. If it does not
418        // recover, the returned status stays `Stalled` and nothing was
419        // clobbered (the recovery only mutates entries for the lost seats).
420        if self.state.status == RunStatus::Stalled {
421            if self.recover_stall().await? {
422                self.finish_after_tally().await?;
423            } else {
424                // Still below quorum: persist the marker and stay resumable.
425                self.state.save()?;
426            }
427            return Ok(());
428        }
429        self.prep().await?;
430        if self.park_here()? {
431            return Ok(());
432        }
433        self.implement().await?;
434        if self.park_here()? {
435            return Ok(());
436        }
437        self.judge().await?;
438        if self.park_here()? {
439            return Ok(());
440        }
441        self.deliberate().await?;
442        if self.park_here()? {
443            return Ok(());
444        }
445        self.vote().await?;
446        if self.park_here()? {
447            return Ok(());
448        }
449        self.tally()?;
450        // A verdict that lost its quorum is not trustworthy: do not review,
451        // gate, or merge on it. Everything already done is kept, so the run
452        // stays resumable (or foldable); the human can replace the agent that
453        // ran out of quota and pick it up.
454        if self.state.status == RunStatus::Stalled {
455            // Persist the stalled marker now — the normal end-of-execute save
456            // below is below this early return, and without it a resumed run
457            // would reload a pre-tally status and keep going.
458            self.state.save()?;
459            return Ok(());
460        }
461        self.finish_after_tally().await?;
462        Ok(())
463    }
464
465    /// Park here if asked to, recording it in the run's own timeline.
466    ///
467    /// Returns whether the caller should stop walking the graph. The state is
468    /// saved either way by the node that just finished; this adds the event so
469    /// the operator's card says why a run that is neither finished nor moving
470    /// is sitting where it is.
471    fn park_here(&mut self) -> Result<bool> {
472        if !self.pause.parked() {
473            return Ok(false);
474        }
475        self.state.event(
476            "park",
477            format!(
478                "parked after `{}` — resume to carry on from here",
479                self.state.status.as_str()
480            ),
481        );
482        self.state.parked = true;
483        self.state.save()?;
484        Ok(true)
485    }
486
487    /// Hand the runner a pause to watch.
488    pub fn on_pause(&mut self, pause: Pause) {
489        self.pause = pause;
490    }
491
492    /// The tail of the graph after a trustworthy tally: fold losers, review,
493    /// gate, merge, and persist.
494    async fn finish_after_tally(&mut self) -> Result<()> {
495        self.fold_losers().await?;
496        // Before review starts, and again right before the gate: a run's
497        // review rounds can themselves take long enough for the base to move
498        // a second time, and the gate is the one node whose "green" gets
499        // acted on.
500        self.sync_to_base().await?;
501        self.review_loop().await?;
502        self.sync_to_base().await?;
503        self.gate().await?;
504        self.merge().await?;
505        self.state.save()?;
506        Ok(())
507    }
508
509    // ---------------------------------------------------------------- prep
510
511    async fn prep(&mut self) -> Result<()> {
512        if !self.state.candidates.is_empty() {
513            return Ok(());
514        }
515        self.state.status = RunStatus::Prep;
516        let repo = self.state.repo.clone();
517        let base = self.state.base_commit.clone();
518        let root = self.state.worktree_root();
519        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
520
521        // The hook is the write-time half of the blindness contract; the
522        // presentation filter in `blind` is the half that cannot be bypassed.
523        let hooks_dir = self.state.dir().join("hooks");
524        if self.state.config.blind.commit_msg_hook {
525            std::fs::create_dir_all(&hooks_dir)
526                .with_context(|| format!("create {}", hooks_dir.display()))?;
527            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
528            let path = hooks_dir.join("commit-msg");
529            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
530            make_executable(&path)?;
531            if git::enable_worktree_config(&repo).await? {
532                self.state.enabled_worktree_config = true;
533            }
534        }
535
536        for (index, (spec, label)) in self
537            .roles
538            .implementers
539            .clone()
540            .into_iter()
541            .zip(labels)
542            .enumerate()
543        {
544            let branch = self.state.branch_for(label);
545            let worktree = root.join(format!("cand-{label}"));
546            git::worktree_add_branch(&repo, &worktree, &branch, &base).await?;
547            if self.state.config.blind.commit_msg_hook {
548                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
549            }
550            git::local_exclude(&worktree, "/.magi/").await?;
551            self.state.candidates.push(Candidate {
552                index,
553                label,
554                agent: spec.id.clone(),
555                branch,
556                worktree,
557                summary: String::new(),
558                stat: String::new(),
559                files: 0,
560                commits: 0,
561                empty: false,
562                failed: None,
563                duration_ms: 0,
564                folded: false,
565            });
566        }
567
568        for j in 1..=self.roles.judges.len() {
569            let wt = root.join(format!("judge-{j}"));
570            if !wt.exists() {
571                git::worktree_add_detached(&repo, &wt, &base).await?;
572            }
573        }
574
575        // A judge cannot tell it is looking at its own patch — the seats keep
576        // separate conversations — but a panel that shares agents with the
577        // field is less independent than it looks, and that is worth saying out
578        // loud once per run rather than leaving it in the config.
579        let authors: Vec<&str> = self
580            .roles
581            .implementers
582            .iter()
583            .map(|a| a.id.as_str())
584            .collect();
585        let overlap: Vec<String> = self
586            .roles
587            .judges
588            .iter()
589            .enumerate()
590            .filter(|(_, j)| authors.contains(&j.id.as_str()))
591            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
592            .collect();
593        if !overlap.is_empty() {
594            let note = format!(
595                "{} also authored a candidate; blind, but the panel is less \
596                 independent than {} distinct agents would be",
597                overlap.join(", "),
598                self.roles.judges.len()
599            );
600            self.state.event("prep", note);
601        }
602
603        self.state.event(
604            "prep",
605            format!(
606                "{} candidates, {} judges, base {} ({})",
607                self.state.candidates.len(),
608                self.roles.judges.len(),
609                &self.state.base_commit[..7.min(self.state.base_commit.len())],
610                self.state.base_branch
611            ),
612        );
613        self.state.status = RunStatus::Implementing;
614        self.state.save()?;
615        Ok(())
616    }
617
618    // ----------------------------------------------------------- implement
619
620    async fn implement(&mut self) -> Result<()> {
621        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
622        // agent files with `magi task add` name the run that paid for it. The
623        // prompt overlay is cloned alongside it because the waves borrow it
624        // while `self` is mutably borrowed by the node's own bookkeeping.
625        let run_id = self.state.id.clone();
626        let prompts = self.state.config.prompts.clone();
627        let todo: Vec<usize> = self
628            .state
629            .candidates
630            .iter()
631            .enumerate()
632            .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
633            .map(|(i, _)| i)
634            .collect();
635        if todo.is_empty() {
636            return self.after_implement();
637        }
638        self.state.status = RunStatus::Implementing;
639
640        let language = self.state.config.graph.language.clone();
641        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
642        let sessions = self.state.config.graph.sessions;
643        let artifacts = agent::artifacts_dir(&self.state.dir());
644
645        let mut jobs = Vec::new();
646        for &i in &todo {
647            let (index, label, worktree) = {
648                let c = &self.state.candidates[i];
649                (c.index, c.label, c.worktree.clone())
650            };
651            let spec = self.roles.implementers[index].clone();
652            let seat_key = format!("impl-{label}");
653            let seat = self.seat(&seat_key, &spec.id);
654            let instruction = self.state.instruction.clone();
655            jobs.push(SeatJob {
656                spec,
657                seat,
658                prompt: prompt::implement(&instruction, &worktree.to_string_lossy(), &language),
659                cwd: worktree,
660                timeout,
661                allow_write: true,
662                sessions,
663                artifacts: artifacts.clone(),
664                stem: format!("impl-{label}"),
665            });
666        }
667
668        self.state.event(
669            "implement",
670            format!("{} candidates in parallel", jobs.len()),
671        );
672        // Kept so a seat whose CLI hung up can be asked again from the same
673        // job: `wave` consumes what it is given.
674        let sent = jobs.clone();
675        let cache = self.state.config.cache_dir();
676        let ctx = WaveCtx {
677            run: &run_id,
678            node: "implement",
679            prompts: &prompts,
680            cache: cache.as_deref(),
681        };
682        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
683        self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
684            .await;
685
686        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
687            let seat_key = seat.key.clone();
688            self.state.seats.insert(seat.key.clone(), seat);
689            let label = self.state.candidates[i].label;
690            let worktree = self.state.candidates[i].worktree.clone();
691            let base = self.state.base_commit.clone();
692
693            let (summary, duration, failed) = match out {
694                AgentOutcome::Ok(o) => {
695                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
696                    let failed = (!o.usable()).then(|| {
697                        if o.timed_out {
698                            "agent timed out".to_owned()
699                        } else {
700                            format!("agent exited with {:?}", o.exit_code)
701                        }
702                    });
703                    (text, o.duration_ms, failed)
704                }
705                // Left un-resumed by `resume_undelivered` (a dirty tree
706                // already rescues the work, or there was no session left to
707                // resume into) — reported like the ordinary failure it is,
708                // never as if `o.text` (the CLI's raw error JSON) were an
709                // answer.
710                AgentOutcome::Dropped(o) => {
711                    let why = o
712                        .dropped
713                        .as_ref()
714                        .map(|d| d.why.as_str())
715                        .unwrap_or("the CLI ended the stream without delivering its answer");
716                    (
717                        String::new(),
718                        o.duration_ms,
719                        Some(format!("the CLI dropped the stream ({why})")),
720                    )
721                }
722                AgentOutcome::Quota(o) => {
723                    self.state.quota.push(QuotaLoss {
724                        seat: seat_key,
725                        node: "implement".to_owned(),
726                        at: Timestamp::now(),
727                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
728                    });
729                    (
730                        String::new(),
731                        o.duration_ms,
732                        Some("rate limited (quota); produced no change".to_owned()),
733                    )
734                }
735                AgentOutcome::Failed(e) => (String::new(), 0, Some(e)),
736            };
737
738            // Rescue anything the agent edited but never committed: an
739            // uncommitted candidate would silently be an empty one.
740            let rescued = git::commit_all(
741                &worktree,
742                &format!("magi: candidate {label} (uncommitted work)"),
743            )
744            .await
745            .unwrap_or(false);
746            let commits = git::commits_ahead(&worktree, &base, "HEAD")
747                .await
748                .unwrap_or(0);
749            let patch = git::diff(&worktree, &base, "HEAD")
750                .await
751                .unwrap_or_default();
752            let stat = git::diff_stat(&worktree, &base, "HEAD")
753                .await
754                .unwrap_or_default();
755            let files = git::changed_files(&worktree, &base, "HEAD")
756                .await
757                .map(|f| f.len())
758                .unwrap_or(0);
759            write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
760
761            let c = &mut self.state.candidates[i];
762            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
763            c.stat = stat;
764            c.files = files;
765            c.commits = commits;
766            c.duration_ms = duration;
767            c.empty = commits == 0 || patch.trim().is_empty();
768            // An agent that failed but still produced a committed change stays
769            // in the running: the patch is what gets judged, not the exit code.
770            c.failed = match failed {
771                Some(_) if c.empty => failed,
772                _ => None,
773            };
774            let note = match (&c.failed, c.empty, rescued) {
775                (Some(e), _, _) => format!("candidate {label}: {e}"),
776                (None, true, _) => format!("candidate {label}: no change produced"),
777                (None, false, true) => {
778                    format!(
779                        "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
780                    )
781                }
782                (None, false, false) => {
783                    format!("candidate {label}: {files} files, {commits} commits")
784                }
785            };
786            self.state.event("implement", note);
787            self.state.save()?;
788        }
789
790        self.after_implement()
791    }
792
793    /// Ask again, once, for work a CLI did and then failed to hand over.
794    ///
795    /// [`agent::dropped_stream`] recognises the one shape observed: an error
796    /// status with an empty response and a usage report showing output tokens,
797    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
798    /// seven minutes and 14,267 output tokens that arrived as an empty
799    /// candidate, because `agy`'s own subscriber fell behind and hung up.
800    ///
801    /// Two conditions, and both matter:
802    ///
803    /// - **Only when the tree is untouched.** Often the agent has already
804    ///   written its files and only the closing message was lost; the rescue
805    ///   commit below picks that up and there is nothing to ask for. Re-asking
806    ///   then would pay for a second implementation of work already on disk.
807    /// - **Once.** A CLI that drops one stream can drop the next, and this
808    ///   node is the most expensive in the graph.
809    ///
810    /// The re-ask is a resume, not a re-run: `has_context` is true because the
811    /// dropped reply still carried its `conversation_id`, so the seat is asked
812    /// to finish what it was doing rather than sent the whole task again. It
813    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
814    /// node's - for the same reason a re-ranked judge does: restating finished
815    /// work is not the work.
816    ///
817    /// Unlike a quota this is worth retrying at all: a rate limit fails the
818    /// same way until it resets, while an abandoned conversation is still
819    /// there to be picked up.
820    async fn resume_undelivered(
821        &mut self,
822        results: &mut [(usize, SeatState, AgentOutcome)],
823        sent: &[SeatJob],
824        prompts: &Prompts,
825        run_id: &str,
826    ) {
827        for (wi, seat, out) in results.iter_mut() {
828            let Some(dropped) = (match &*out {
829                AgentOutcome::Dropped(o) => o.dropped.clone(),
830                _ => None,
831            }) else {
832                continue;
833            };
834            let Some(job) = sent.get(*wi) else { continue };
835            // Already on disk? Then only the closing message was lost.
836            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
837                self.state.event(
838                    "implement",
839                    format!(
840                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
841                         work is in the tree",
842                        seat.key, dropped.output_tokens, dropped.why
843                    ),
844                );
845                continue;
846            }
847            // The re-ask only makes sense as a resume: `resume_after_drop`
848            // says nothing about the task, trusting the seat to still hold it.
849            // Without a session to resume — sessions disabled, or this CLI's
850            // drop shape happened not to carry a session id — that prompt
851            // would open a brand-new conversation with no context at all,
852            // which is worse than leaving this as the ordinary failure it
853            // already is.
854            if !has_context(&job.spec, seat, job.sessions) {
855                self.state.event(
856                    "implement",
857                    format!(
858                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
859                         is no session left to resume",
860                        seat.key, dropped.output_tokens, dropped.why
861                    ),
862                );
863                continue;
864            }
865            self.state.event(
866                "implement",
867                format!(
868                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
869                     conversation",
870                    seat.key, dropped.output_tokens, dropped.why
871                ),
872            );
873            let mut retry = job.clone();
874            retry.seat = seat.clone();
875            retry.prompt = prompt::resume_after_drop(&dropped.why);
876            retry.timeout = retry_budget(job.timeout, true);
877            retry.stem = format!("{}-resume", job.stem);
878            let cache = self.state.config.cache_dir();
879            let ctx = WaveCtx {
880                run: run_id,
881                node: "implement",
882                prompts,
883                cache: cache.as_deref(),
884            };
885            let (resumed_seat, resumed) =
886                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
887            *seat = resumed_seat;
888            *out = resumed;
889        }
890    }
891
892    fn after_implement(&mut self) -> Result<()> {
893        // Scan every candidate patch once the set is complete.
894        if self.state.leaks.is_empty() {
895            let cfg = self.state.config.blind.clone();
896            let mut leaks = Vec::new();
897            for c in &self.state.candidates {
898                let Some(patch) =
899                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
900                else {
901                    continue;
902                };
903                leaks.extend(blind::scan(
904                    &format!("candidate {} patch", c.label),
905                    &patch,
906                    &cfg.vendor_tokens,
907                ));
908            }
909            if !leaks.is_empty() {
910                let summary = leaks
911                    .iter()
912                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
913                    .collect::<Vec<_>>()
914                    .join(", ");
915                match cfg.on_leak {
916                    LeakPolicy::Fail => {
917                        self.state.status = RunStatus::Failed;
918                        self.state
919                            .event("blind", format!("vendor text in a patch: {summary}"));
920                        self.state.leaks = leaks;
921                        self.state.save()?;
922                        bail!(
923                            "blind.on_leak = \"fail\" and vendor text reached a \
924                             judged patch: {summary}"
925                        );
926                    }
927                    LeakPolicy::Redact => self.state.event(
928                        "blind",
929                        format!("redacting vendor text for judging: {summary}"),
930                    ),
931                    LeakPolicy::Warn => self.state.event(
932                        "blind",
933                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
934                    ),
935                }
936                self.state.leaks = leaks;
937            }
938        }
939
940        if self.state.viable().is_empty() {
941            self.state.status = RunStatus::Failed;
942            self.state.save()?;
943            bail!("no candidate produced a change; nothing to judge");
944        }
945        self.state.status = RunStatus::Judging;
946        self.state.save()?;
947        Ok(())
948    }
949
950    // --------------------------------------------------------------- judge
951
952    async fn judge(&mut self) -> Result<()> {
953        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
954        // agent files with `magi task add` name the run that paid for it. The
955        // prompt overlay is cloned alongside it because the waves borrow it
956        // while `self` is mutably borrowed by the node's own bookkeeping.
957        let run_id = self.state.id.clone();
958        let prompts = self.state.config.prompts.clone();
959        if !self.state.judgements.is_empty() || self.state.judge_skipped {
960            return Ok(());
961        }
962        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
963        if viable.len() == 1 {
964            // Recorded so this is a one-time event: `judgements` stays empty
965            // either way, which without this flag is indistinguishable from
966            // "not yet judged" on the next reentry — and status is left
967            // untouched, so a later node's conclusion (e.g. `Blocked` after
968            // the review budget ran out) survives a resume instead of being
969            // clobbered back to `Judging` by this node running again.
970            self.state.judge_skipped = true;
971            self.state.event(
972                "judge",
973                format!(
974                    "only candidate {} produced a change; judging skipped",
975                    viable[0].label
976                ),
977            );
978            self.state.save()?;
979            return Ok(());
980        }
981        self.state.status = RunStatus::Judging;
982
983        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
984        let language = self.state.config.graph.language.clone();
985        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
986        let sessions = self.state.config.graph.sessions;
987        let artifacts = agent::artifacts_dir(&self.state.dir());
988        let root = self.state.worktree_root();
989        let base_short = short(&self.state.base_commit);
990
991        let mut jobs = Vec::new();
992        let mut orders = Vec::new();
993        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
994            let order = blind::presentation_order(viable.len(), j, self.state.seed);
995            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
996            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
997            let seat_key = format!("judge-{}", j + 1);
998            let seat = self.seat(&seat_key, &spec.id);
999            jobs.push(SeatJob {
1000                prompt: prompt::judge(
1001                    &self.state.instruction,
1002                    &views,
1003                    self.roles.judges.len(),
1004                    &base_short,
1005                    &language,
1006                ),
1007                spec,
1008                seat,
1009                cwd: root.join(format!("judge-{}", j + 1)),
1010                timeout,
1011                allow_write: false,
1012                sessions,
1013                artifacts: artifacts.clone(),
1014                stem: format!("judge-{}", j + 1),
1015            });
1016        }
1017
1018        self.state.event(
1019            "judge",
1020            format!(
1021                "{} judges ranking {} candidates blind",
1022                jobs.len(),
1023                viable.len()
1024            ),
1025        );
1026        let labels_for_check = labels.clone();
1027        let mut quota_losses = Vec::new();
1028        let cache = self.state.config.cache_dir();
1029        let ctx = WaveCtx {
1030            run: &run_id,
1031            node: "judge",
1032            prompts: &prompts,
1033            cache: cache.as_deref(),
1034        };
1035        let results = ask_json_wave::<Ranking>(
1036            jobs,
1037            Arc::clone(&self.sem),
1038            self.state.config.graph.retries,
1039            &ctx,
1040            &mut quota_losses,
1041            &mut self.state,
1042            &move |r: &Ranking| r.validate(&labels_for_check),
1043        )
1044        .await;
1045        self.state.quota.extend(quota_losses);
1046
1047        for (j, (seat, res)) in results.into_iter().enumerate() {
1048            let agent_id = seat.agent.clone();
1049            self.state.seats.insert(seat.key.clone(), seat);
1050            let mut record = Judgement {
1051                judge: j + 1,
1052                seat: format!("judge-{}", j + 1),
1053                agent: agent_id,
1054                ranking: Vec::new(),
1055                reasons: BTreeMap::new(),
1056                confidence: None,
1057                order: orders[j].clone(),
1058                failed: None,
1059                duration_ms: 0,
1060            };
1061            match res {
1062                Ok((ranking, out)) => {
1063                    record.ranking = ranking.normalized();
1064                    record.reasons = ranking.reasons;
1065                    record.confidence = ranking.confidence;
1066                    record.duration_ms = out.duration_ms;
1067                    self.state.event(
1068                        "judge",
1069                        format!(
1070                            "judge {} ranked {}",
1071                            j + 1,
1072                            record.ranking.iter().collect::<String>()
1073                        ),
1074                    );
1075                }
1076                Err(e) => {
1077                    record.failed = Some(e.to_string());
1078                    self.state
1079                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
1080                }
1081            }
1082            self.state.judgements.push(record);
1083            self.state.save()?;
1084        }
1085        Ok(())
1086    }
1087
1088    // ---------------------------------------------------------- deliberate
1089
1090    async fn deliberate(&mut self) -> Result<()> {
1091        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1092        // agent files with `magi task add` name the run that paid for it. The
1093        // prompt overlay is cloned alongside it because the waves borrow it
1094        // while `self` is mutably borrowed by the node's own bookkeeping.
1095        let run_id = self.state.id.clone();
1096        let prompts = self.state.config.prompts.clone();
1097        if !self.state.deliberation.is_empty() {
1098            return Ok(());
1099        }
1100        let tops: Vec<char> = self
1101            .state
1102            .judgements
1103            .iter()
1104            .filter_map(|j| j.ranking.first().copied())
1105            .collect();
1106        let rounds = self.state.config.graph.deliberate_rounds;
1107        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
1108            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
1109                self.state.event(
1110                    "deliberate",
1111                    format!("judges agreed on {} outright; no deliberation", tops[0]),
1112                );
1113            }
1114            self.state.status = RunStatus::Voting;
1115            self.state.save()?;
1116            return Ok(());
1117        }
1118
1119        self.state.status = RunStatus::Deliberating;
1120        self.state.event(
1121            "deliberate",
1122            format!(
1123                "split: first choices were {} — opening {rounds} round(s)",
1124                tops.iter().collect::<String>()
1125            ),
1126        );
1127
1128        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1129        let language = self.state.config.graph.language.clone();
1130        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1131        let sessions = self.state.config.graph.sessions;
1132        let artifacts = agent::artifacts_dir(&self.state.dir());
1133        let root = self.state.worktree_root();
1134        let base_short = short(&self.state.base_commit);
1135
1136        // Judges argue in sequence so that a turn can answer the one before it;
1137        // that is the difference between deliberation and three parallel
1138        // monologues.
1139        for round in 1..=rounds {
1140            let mut turns: Vec<DeliberationTurn> = Vec::new();
1141            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1142                if self.state.judgements[j].failed.is_some() {
1143                    continue;
1144                }
1145                let seat_key = format!("judge-{}", j + 1);
1146                let mut seat = self.seat(&seat_key, &spec.id);
1147                let transcript = self.transcript(&turns, j);
1148                let context = if has_context(&spec, &seat, sessions) {
1149                    None
1150                } else {
1151                    Some(self.candidate_block(&viable, &base_short))
1152                };
1153                let text = prompt::deliberate(
1154                    &self.state.instruction,
1155                    context.as_deref(),
1156                    &transcript,
1157                    round,
1158                    rounds,
1159                    &language,
1160                );
1161                let job = SeatJob {
1162                    spec,
1163                    seat: seat.clone(),
1164                    prompt: text,
1165                    cwd: root.join(format!("judge-{}", j + 1)),
1166                    timeout,
1167                    allow_write: false,
1168                    sessions,
1169                    artifacts: artifacts.clone(),
1170                    stem: format!("delib-{round}-judge-{}", j + 1),
1171                };
1172                let cache = self.state.config.cache_dir();
1173                let ctx = WaveCtx {
1174                    run: &run_id,
1175                    node: "deliberate",
1176                    prompts: &prompts,
1177                    cache: cache.as_deref(),
1178                };
1179                let (updated, out) =
1180                    run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1181                seat = updated;
1182                let agent_id = seat.agent.clone();
1183                let seat_key = seat.key.clone();
1184                self.state.seats.insert(seat.key.clone(), seat);
1185                let body = match out {
1186                    AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
1187                    // Never read the CLI's raw error JSON as this judge's
1188                    // position — skip the seat instead, the same as any other
1189                    // failed turn.
1190                    AgentOutcome::Dropped(o) => {
1191                        let why =
1192                            o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
1193                                "the CLI ended the stream without delivering its answer",
1194                            );
1195                        self.state.event(
1196                            "deliberate",
1197                            format!(
1198                                "judge {} skipped: the CLI dropped the stream ({why})",
1199                                j + 1
1200                            ),
1201                        );
1202                        continue;
1203                    }
1204                    AgentOutcome::Quota(o) => {
1205                        self.state.quota.push(QuotaLoss {
1206                            seat: seat_key,
1207                            node: "deliberate".to_owned(),
1208                            at: Timestamp::now(),
1209                            reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1210                        });
1211                        self.state.event(
1212                            "deliberate",
1213                            format!("judge {} skipped: rate limited (quota)", j + 1),
1214                        );
1215                        continue;
1216                    }
1217                    AgentOutcome::Failed(e) => {
1218                        self.state
1219                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
1220                        continue;
1221                    }
1222                };
1223                let tentative = verdict::extract_json::<Position>(&body)
1224                    .ok()
1225                    .and_then(|p| p.tentative)
1226                    .and_then(|s| s.trim().chars().next())
1227                    .map(|c| c.to_ascii_uppercase());
1228                self.state.event(
1229                    "deliberate",
1230                    format!(
1231                        "round {round}: judge {} now favours {}",
1232                        j + 1,
1233                        tentative.map_or("—".to_owned(), |c| c.to_string())
1234                    ),
1235                );
1236                turns.push(DeliberationTurn {
1237                    judge: j + 1,
1238                    agent: agent_id,
1239                    body: blind::sanitize_prose(&body, &self.state.config.blind),
1240                    tentative,
1241                });
1242            }
1243            self.state
1244                .deliberation
1245                .push(DeliberationRound { round, turns });
1246            self.state.save()?;
1247        }
1248
1249        self.state.status = RunStatus::Voting;
1250        self.state.save()?;
1251        Ok(())
1252    }
1253
1254    // ---------------------------------------------------------------- vote
1255
1256    async fn vote(&mut self) -> Result<()> {
1257        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1258        // agent files with `magi task add` name the run that paid for it. The
1259        // prompt overlay is cloned alongside it because the waves borrow it
1260        // while `self` is mutably borrowed by the node's own bookkeeping.
1261        let run_id = self.state.id.clone();
1262        let prompts = self.state.config.prompts.clone();
1263        if !self.state.votes.is_empty() {
1264            return Ok(());
1265        }
1266        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
1267        if viable.len() == 1 {
1268            return Ok(());
1269        }
1270        self.state.status = RunStatus::Voting;
1271
1272        let language = self.state.config.graph.language.clone();
1273        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1274        let sessions = self.state.config.graph.sessions;
1275        let artifacts = agent::artifacts_dir(&self.state.dir());
1276        let root = self.state.worktree_root();
1277        let base_short = short(&self.state.base_commit);
1278        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1279
1280        let mut jobs = Vec::new();
1281        let mut seats_at = Vec::new();
1282        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1283            if self
1284                .state
1285                .judgements
1286                .get(j)
1287                .is_some_and(|r| r.failed.is_some())
1288            {
1289                continue;
1290            }
1291            let seat_key = format!("judge-{}", j + 1);
1292            let seat = self.seat(&seat_key, &spec.id);
1293            let mut text = prompt::final_vote(&viable, &language);
1294            if !has_context(&spec, &seat, sessions) {
1295                text = format!(
1296                    "{}\n\n# Candidates\n\n{}",
1297                    text,
1298                    self.candidate_block(&candidates, &base_short)
1299                );
1300            }
1301            jobs.push(SeatJob {
1302                spec,
1303                seat,
1304                prompt: text,
1305                cwd: root.join(format!("judge-{}", j + 1)),
1306                timeout,
1307                allow_write: false,
1308                sessions,
1309                artifacts: artifacts.clone(),
1310                stem: format!("vote-judge-{}", j + 1),
1311            });
1312            seats_at.push(j);
1313        }
1314
1315        self.state.event(
1316            "vote",
1317            format!(
1318                "collecting {} final votes one by one, privately",
1319                jobs.len()
1320            ),
1321        );
1322        let allowed = viable.clone();
1323        let mut quota_losses = Vec::new();
1324        let cache = self.state.config.cache_dir();
1325        let ctx = WaveCtx {
1326            run: &run_id,
1327            node: "vote",
1328            prompts: &prompts,
1329            cache: cache.as_deref(),
1330        };
1331        let results = ask_json_wave::<FinalVote>(
1332            jobs,
1333            Arc::clone(&self.sem),
1334            self.state.config.graph.retries,
1335            &ctx,
1336            &mut quota_losses,
1337            &mut self.state,
1338            &move |v: &FinalVote| match v.label() {
1339                Some(c) if allowed.contains(&c) => Ok(()),
1340                other => bail!("vote {other:?} is not one of {allowed:?}"),
1341            },
1342        )
1343        .await;
1344        self.state.quota.extend(quota_losses);
1345
1346        for (&j, (seat, res)) in seats_at.iter().zip(results) {
1347            let agent_id = seat.agent.clone();
1348            self.state.seats.insert(seat.key.clone(), seat);
1349            let initial = self
1350                .state
1351                .judgements
1352                .get(j)
1353                .and_then(|r| r.ranking.first().copied());
1354            let mut record = VoteRecord {
1355                judge: j + 1,
1356                agent: agent_id,
1357                vote: None,
1358                reason: String::new(),
1359                changed: false,
1360            };
1361            match res {
1362                Ok((v, _)) => {
1363                    record.vote = v.label();
1364                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
1365                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
1366                    self.state.event(
1367                        "vote",
1368                        format!(
1369                            "judge {} voted {}{}",
1370                            j + 1,
1371                            record.vote.unwrap_or('?'),
1372                            if record.changed { " (changed)" } else { "" }
1373                        ),
1374                    );
1375                }
1376                Err(e) => {
1377                    self.state
1378                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
1379                }
1380            }
1381            self.state.votes.push(record);
1382            self.state.save()?;
1383        }
1384        Ok(())
1385    }
1386
1387    // --------------------------------------------------------------- tally
1388
1389    fn tally(&mut self) -> Result<()> {
1390        if self.state.tally.is_some() {
1391            return Ok(());
1392        }
1393        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
1394        let tops: Vec<char> = self
1395            .state
1396            .judgements
1397            .iter()
1398            .filter_map(|j| j.ranking.first().copied())
1399            .collect();
1400        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
1401
1402        // A judge whose private vote failed still counted once, in the initial
1403        // ranking; using it beats discarding a whole seat.
1404        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
1405        let mut cast: Vec<char> = Vec::new();
1406        for (i, j) in self.state.judgements.iter().enumerate() {
1407            let vote = self
1408                .state
1409                .votes
1410                .iter()
1411                .find(|v| v.judge == i + 1)
1412                .and_then(|v| v.vote)
1413                .or_else(|| j.ranking.first().copied());
1414            if let Some(v) = vote {
1415                *first_choice.entry(v).or_insert(0) += 1;
1416                cast.push(v);
1417            }
1418        }
1419
1420        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
1421        for j in &self.state.judgements {
1422            let n = j.ranking.len();
1423            for (pos, label) in j.ranking.iter().enumerate() {
1424                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
1425            }
1426        }
1427
1428        let best = first_choice.values().copied().max().unwrap_or(0);
1429        let mut leaders: Vec<char> = first_choice
1430            .iter()
1431            .filter(|(_, v)| **v == best)
1432            .map(|(k, _)| *k)
1433            .collect();
1434        let mut tie_break = None;
1435        if leaders.len() > 1 {
1436            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
1437            let borda_leaders: Vec<char> = leaders
1438                .iter()
1439                .copied()
1440                .filter(|l| borda[l] == top_borda)
1441                .collect();
1442            tie_break = Some(if borda_leaders.len() == 1 {
1443                format!(
1444                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
1445                    leaders.len()
1446                )
1447            } else {
1448                format!(
1449                    "{} way tie on both first-choice votes and Borda points, broken by label order",
1450                    leaders.len()
1451                )
1452            });
1453            leaders = borda_leaders;
1454            leaders.sort_unstable();
1455        }
1456        let winner = *leaders
1457            .first()
1458            .or(viable.first())
1459            .context("no candidate to declare a winner from")?;
1460
1461        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
1462        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
1463        let deliberated = !self.state.deliberation.is_empty();
1464
1465        // Whose verdict is this? A rate-limited seat is absent even if it
1466        // ranked before the limit hit, so presence is measured against the
1467        // recorded losses, not just "did a ranking ever appear".
1468        let quota_seats: std::collections::BTreeSet<&str> =
1469            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
1470        let mut present = 0usize;
1471        for (i, j) in self.state.judgements.iter().enumerate() {
1472            if quota_seats.contains(j.seat.as_str()) {
1473                continue;
1474            }
1475            let ranked = !j.ranking.is_empty() && j.failed.is_none();
1476            let voted = self
1477                .state
1478                .votes
1479                .iter()
1480                .any(|v| v.judge == i + 1 && v.vote.is_some());
1481            if ranked || voted {
1482                present += 1;
1483            }
1484        }
1485        // Strict majority of the configured panel. A bare majority is real
1486        // signal we can act on, while a minority verdict must never stand in
1487        // for a healthy one. A one-candidate run needs no panel at all, and
1488        // `judges` stays `0` rather than the roster size a panel that never
1489        // sat would otherwise be credited with.
1490        let needs_quorum = viable.len() > 1;
1491        let judges_total = if needs_quorum {
1492            self.roles.judges.len()
1493        } else {
1494            0
1495        };
1496        let quorum = if needs_quorum {
1497            judges_total / 2 + 1
1498        } else {
1499            0
1500        };
1501        let met_quorum = !needs_quorum || present >= quorum;
1502        let uncontested = (!needs_quorum).then(|| {
1503            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
1504        });
1505
1506        self.state.event(
1507            "tally",
1508            match &uncontested {
1509                Some(reason) => format!("winner {winner} — {reason}"),
1510                None => format!(
1511                    "winner {winner} — votes {} | initial {} | {} changed | \
1512                     {present}/{judges_total} judges{}",
1513                    first_choice
1514                        .iter()
1515                        .map(|(k, v)| format!("{k}:{v}"))
1516                        .collect::<Vec<_>>()
1517                        .join(" "),
1518                    if unanimous_initial {
1519                        "unanimous"
1520                    } else {
1521                        "split"
1522                    },
1523                    changed_votes,
1524                    if met_quorum {
1525                        String::new()
1526                    } else {
1527                        format!(" — below quorum ({quorum} required)")
1528                    },
1529                ),
1530            },
1531        );
1532        if !met_quorum {
1533            self.state.event(
1534                "stall",
1535                format!(
1536                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
1537                     the run stops here, resumable"
1538                ),
1539            );
1540        }
1541        self.state.tally = Some(Tally {
1542            first_choice,
1543            borda,
1544            winner,
1545            rankings: tops.len(),
1546            unanimous_initial,
1547            deliberated,
1548            changed_votes,
1549            unanimous_final,
1550            tie_break,
1551            judges: judges_total,
1552            present,
1553            quorum,
1554            met_quorum,
1555            uncontested,
1556        });
1557        self.state.status = if met_quorum {
1558            RunStatus::Reviewing
1559        } else {
1560            RunStatus::Stalled
1561        };
1562        self.state.save()?;
1563        Ok(())
1564    }
1565
1566    // ------------------------------------------------------------- recover
1567
1568    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
1569    /// resumed toward completion once the transient cause clears.
1570    ///
1571    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
1572    /// it toward the quorum, which is exactly the set of seats whose absence
1573    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
1574    /// not depend on which node happened to hit the limit), or an ordinary
1575    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
1576    /// seat is never disturbed.
1577    ///
1578    /// A seat that now answers with a usable ranking is "recovered": its
1579    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
1580    /// `tally` counts it present again), and its vote re-collected. A seat that
1581    /// still fails keeps its loss and stays absent.
1582    ///
1583    /// Returns `true` when the re-tally restores the quorum (the run may proceed
1584    /// to review/gate/merge), `false` when it is still below quorum (the run
1585    /// stays `Stalled`, still resumable for a later retry).
1586    #[allow(clippy::too_many_lines)]
1587    async fn recover_stall(&mut self) -> Result<bool> {
1588        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1589        // agent files with `magi task add` name the run that paid for it. The
1590        // prompt overlay is cloned alongside it because the waves borrow it
1591        // while `self` is mutably borrowed by the node's own bookkeeping.
1592        let run_id = self.state.id.clone();
1593        let prompts = self.state.config.prompts.clone();
1594        // Absent seats = quota-lost at any node, or failed outright. Mirroring
1595        // `tally`'s presence test (rather than the old quota-judge/vote filter)
1596        // is what keeps a non-quota collapse — or a quota loss recorded at the
1597        // deliberate node — from being a permanent dead-end on `--resume`.
1598        let quota_seats: BTreeSet<&str> =
1599            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
1600        let absent: Vec<String> = self
1601            .state
1602            .judgements
1603            .iter()
1604            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
1605            .map(|j| j.seat.clone())
1606            .collect();
1607        if absent.is_empty() {
1608            return Ok(false);
1609        }
1610        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1611        if viable.len() <= 1 {
1612            return Ok(false);
1613        }
1614        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
1615        let language = self.state.config.graph.language.clone();
1616        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1617        let sessions = self.state.config.graph.sessions;
1618        let artifacts = agent::artifacts_dir(&self.state.dir());
1619        let root = self.state.worktree_root();
1620        let base_short = short(&self.state.base_commit);
1621        let candidates: Vec<Candidate> = viable.clone();
1622
1623        // Map each absent seat key to its 0-based position in `roles.judges`.
1624        let mut positions: Vec<usize> = absent
1625            .iter()
1626            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
1627            .collect();
1628        if positions.is_empty() {
1629            return Ok(false);
1630        }
1631        positions.sort_unstable();
1632        positions.dedup();
1633
1634        // Re-rank the lost seats, one blind prompt each.
1635        let mut judge_jobs = Vec::new();
1636        for &j in &positions {
1637            let order = blind::presentation_order(viable.len(), j, self.state.seed);
1638            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
1639            let seat_key = format!("judge-{}", j + 1);
1640            let spec = self.roles.judges[j].clone();
1641            let seat = self.seat(&seat_key, &spec.id);
1642            judge_jobs.push(SeatJob {
1643                spec,
1644                seat,
1645                prompt: prompt::judge(
1646                    &self.state.instruction,
1647                    &views,
1648                    self.roles.judges.len(),
1649                    &base_short,
1650                    &language,
1651                ),
1652                cwd: root.join(seat_key),
1653                timeout,
1654                allow_write: false,
1655                sessions,
1656                artifacts: artifacts.clone(),
1657                stem: format!("judge-{}-recover", j + 1),
1658            });
1659        }
1660
1661        let labels_for_check = labels.clone();
1662        let mut judge_losses = Vec::new();
1663        let retries = self.state.config.graph.retries;
1664        let cache = self.state.config.cache_dir();
1665        let ctx = WaveCtx {
1666            run: &run_id,
1667            node: "judge",
1668            prompts: &prompts,
1669            cache: cache.as_deref(),
1670        };
1671        let results = ask_json_wave::<Ranking>(
1672            judge_jobs,
1673            Arc::clone(&self.sem),
1674            retries,
1675            &ctx,
1676            &mut judge_losses,
1677            &mut self.state,
1678            &move |r: &Ranking| r.validate(&labels_for_check),
1679        )
1680        .await;
1681
1682        // Refresh the judgement of every seat that ranked again.
1683        let mut recovered: BTreeSet<usize> = BTreeSet::new();
1684        for (&j, (seat, res)) in positions.iter().zip(results) {
1685            self.state.seats.insert(seat.key.clone(), seat);
1686            let record = &mut self.state.judgements[j];
1687            match res {
1688                Ok((ranking, out)) => {
1689                    record.ranking = ranking.normalized();
1690                    record.reasons = ranking.reasons;
1691                    record.confidence = ranking.confidence;
1692                    record.failed = None;
1693                    record.duration_ms = out.duration_ms;
1694                    recovered.insert(j);
1695                    self.state.event(
1696                        "recover",
1697                        format!("judge {} ranked again after the limit", j + 1),
1698                    );
1699                }
1700                Err(e) => {
1701                    self.state
1702                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
1703                }
1704            }
1705        }
1706
1707        // Re-ask the votes of the seats that recovered a ranking.
1708        let mut vote_jobs = Vec::new();
1709        let mut vote_pos: Vec<usize> = Vec::new();
1710        for &j in &recovered {
1711            let seat_key = format!("judge-{}", j + 1);
1712            let spec = self.roles.judges[j].clone();
1713            let seat = self.seat(&seat_key, &spec.id);
1714            let mut text = prompt::final_vote(&labels, &language);
1715            if !has_context(&spec, &seat, sessions) {
1716                text = format!(
1717                    "{}\n\n# Candidates\n\n{}",
1718                    text,
1719                    self.candidate_block(&candidates, &base_short)
1720                );
1721            }
1722            vote_jobs.push(SeatJob {
1723                spec,
1724                seat,
1725                prompt: text,
1726                cwd: root.join(seat_key),
1727                timeout,
1728                allow_write: false,
1729                sessions,
1730                artifacts: artifacts.clone(),
1731                stem: format!("vote-judge-{}-recover", j + 1),
1732            });
1733            vote_pos.push(j);
1734        }
1735        let allowed = labels.clone();
1736        let mut vote_losses = Vec::new();
1737        let vote_retries = self.state.config.graph.retries;
1738        let vote_cache = self.state.config.cache_dir();
1739        let ctx = WaveCtx {
1740            run: &run_id,
1741            node: "vote",
1742            prompts: &prompts,
1743            cache: vote_cache.as_deref(),
1744        };
1745        let votes = ask_json_wave::<FinalVote>(
1746            vote_jobs,
1747            Arc::clone(&self.sem),
1748            vote_retries,
1749            &ctx,
1750            &mut vote_losses,
1751            &mut self.state,
1752            &move |v: &FinalVote| match v.label() {
1753                Some(c) if allowed.contains(&c) => Ok(()),
1754                other => bail!("vote {other:?} is not one of {allowed:?}"),
1755            },
1756        )
1757        .await;
1758        for (&j, (seat, res)) in vote_pos.iter().zip(votes) {
1759            let agent_id = seat.agent.clone();
1760            self.state.seats.insert(seat.key.clone(), seat);
1761            match res {
1762                Ok((v, _)) => {
1763                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
1764                        rec.vote = v.label();
1765                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
1766                    } else {
1767                        self.state.votes.push(VoteRecord {
1768                            judge: j + 1,
1769                            agent: agent_id,
1770                            vote: v.label(),
1771                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
1772                            changed: false,
1773                        });
1774                    }
1775                    self.state.event(
1776                        "recover",
1777                        format!("judge {} voted again after the limit", j + 1),
1778                    );
1779                }
1780                Err(e) => {
1781                    self.state
1782                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
1783                }
1784            }
1785        }
1786
1787        // A seat that ranked again is present even if its re-vote failed —
1788        // `tally` falls back to the initial ranking's first choice — so clear
1789        // its quota loss. Seats that still fail keep theirs and stay absent.
1790        if !recovered.is_empty() {
1791            let recovered_keys: BTreeSet<String> = recovered
1792                .iter()
1793                .map(|&j| format!("judge-{}", j + 1))
1794                .collect();
1795            self.state
1796                .quota
1797                .retain(|q| !recovered_keys.contains(&q.seat));
1798        }
1799
1800        // Recompute the verdict from the refreshed panel.
1801        self.state.tally = None;
1802        self.tally()?;
1803        Ok(self
1804            .state
1805            .tally
1806            .as_ref()
1807            .map(|t| t.met_quorum)
1808            .unwrap_or(false))
1809    }
1810
1811    // ----------------------------------------------------------------- fold
1812
1813    async fn fold_losers(&mut self) -> Result<()> {
1814        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
1815            return Ok(());
1816        };
1817        let repo = self.state.repo.clone();
1818        let mut folded = Vec::new();
1819        for i in 0..self.state.candidates.len() {
1820            let c = &self.state.candidates[i];
1821            if c.label == winner || c.folded {
1822                continue;
1823            }
1824            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
1825            git::worktree_remove(&repo, &wt).await.ok();
1826            git::branch_delete(&repo, &branch).await.ok();
1827            self.state.candidates[i].folded = true;
1828            folded.push(label.to_string());
1829        }
1830        // The judges are finished; their checkouts are pure cost from here.
1831        let root = self.state.worktree_root();
1832        for j in 1..=self.roles.judges.len() {
1833            let wt = root.join(format!("judge-{j}"));
1834            if wt.exists() {
1835                git::worktree_remove(&repo, &wt).await.ok();
1836            }
1837        }
1838        if !folded.is_empty() {
1839            self.state
1840                .event("fold", format!("folded candidates {}", folded.join(", ")));
1841            self.state.save()?;
1842        }
1843        Ok(())
1844    }
1845
1846    // ------------------------------------------------------------ base sync
1847
1848    /// Land the winner's tree on the current tip of `<remote>/<base>` before
1849    /// anything verifies it.
1850    ///
1851    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
1852    /// read whatever is checked out in the winner's worktree. Left alone that
1853    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
1854    /// it when the run *branched* - and a run takes long enough that the base
1855    /// has usually moved by the time it gets here. A gate that ran there
1856    /// answers "green on the commit this run started from", not "green on
1857    /// what is about to land", and the difference showed up three times in
1858    /// one day as a green run whose merge would have reverted a file another
1859    /// pull request had already landed.
1860    ///
1861    /// Reuses [`git::rebase_branch_in_temp`] rather than a second
1862    /// implementation of the same idea: `land::Step::Rebase` already worked
1863    /// out the rules - throwaway worktree, conflict stops and reports rather
1864    /// than feeding a fixer, nothing runs in the primary tree - and a second
1865    /// rebase path is exactly the kind of drift `resolve_base`'s own doc
1866    /// warns about ("two answers to a question nobody notices until a diff is
1867    /// wrong").
1868    ///
1869    /// Bounded by [`BASE_SYNC_ROUNDS`], counted in `state.base_sync.attempts`
1870    /// so it survives a park/resume. A conflict or a push failure sets
1871    /// `state.base_sync.conflict` and leaves the branch and worktree exactly
1872    /// as they were - untouched, for a person to look at - which is also what
1873    /// makes re-entering this function afterwards a no-op instead of a second
1874    /// attempt at the same wall.
1875    async fn sync_to_base(&mut self) -> Result<()> {
1876        if self
1877            .state
1878            .base_sync
1879            .as_ref()
1880            .is_some_and(|s| s.conflict.is_some())
1881        {
1882            return Ok(());
1883        }
1884        let Some(winner) = self.state.winner().cloned() else {
1885            return Ok(());
1886        };
1887
1888        let repo = self.state.repo.clone();
1889        let remote = self.state.config.merge.remote.clone();
1890        let base_branch = self.state.base_branch.clone();
1891        let tracking = format!("{remote}/{base_branch}");
1892
1893        git::fetch(&repo, &remote, &base_branch).await.ok();
1894        // No network, or the remote never had this branch: `resolve_base`
1895        // already treats that as non-fatal at branch time, and a run that got
1896        // this far must not be blocked by it here either.
1897        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
1898            return Ok(());
1899        };
1900
1901        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1902        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
1903        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
1904
1905        if behind == 0 {
1906            self.state.base_sync = Some(BaseSync {
1907                tip,
1908                behind: 0,
1909                attempts,
1910                conflict: None,
1911            });
1912            self.state.save()?;
1913            return Ok(());
1914        }
1915
1916        if attempts >= BASE_SYNC_ROUNDS {
1917            let why = format!(
1918                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
1919                 rebase(s); rebasing again would only race it",
1920                winner.branch
1921            );
1922            self.state.status = RunStatus::Blocked;
1923            self.state.base_sync = Some(BaseSync {
1924                tip,
1925                behind,
1926                attempts,
1927                conflict: Some(why.clone()),
1928            });
1929            self.state.event("land", why);
1930            self.state.save()?;
1931            return Ok(());
1932        }
1933
1934        self.state.event(
1935            "land",
1936            format!(
1937                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
1938                winner.branch
1939            ),
1940        );
1941        self.state.save()?;
1942
1943        let scratch = self.state.dir().join("base-sync");
1944        let rebased = git::rebase_branch_in_temp(&repo, &scratch, &winner.branch, &tracking).await;
1945        let attempts = attempts + 1;
1946        match rebased {
1947            Ok(None) => {
1948                // The branch ref moved, but a worktree that already had it
1949                // checked out (the winner's) was not told; sync its index and
1950                // files before anything reads them.
1951                git::sync_to_head(&winner.worktree).await?;
1952                self.state.base_sync = Some(BaseSync {
1953                    tip: tip.clone(),
1954                    behind: 0,
1955                    attempts,
1956                    conflict: None,
1957                });
1958                self.state
1959                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
1960            }
1961            Ok(Some(conflict)) => {
1962                let why = format!(
1963                    "{} conflicts with {tracking} and did not rebase: {}",
1964                    winner.branch,
1965                    conflict.chars().take(600).collect::<String>()
1966                );
1967                self.state.status = RunStatus::Blocked;
1968                self.state.base_sync = Some(BaseSync {
1969                    tip,
1970                    behind,
1971                    attempts,
1972                    conflict: Some(why.clone()),
1973                });
1974                self.state.event("land", why);
1975            }
1976            Err(e) => {
1977                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
1978                self.state.status = RunStatus::Blocked;
1979                self.state.base_sync = Some(BaseSync {
1980                    tip,
1981                    behind,
1982                    attempts,
1983                    conflict: Some(why.clone()),
1984                });
1985                self.state.event("land", why);
1986            }
1987        }
1988        self.state.save()?;
1989        Ok(())
1990    }
1991
1992    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
1993    /// last landed the winner on, once it has run, else the commit the run
1994    /// branched from.
1995    ///
1996    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
1997    /// and `vote` all happen before there is a winner to rebase, so they
1998    /// compare every candidate against the branch point on purpose, and a
1999    /// base that moves after they are already done cannot change an answer
2000    /// they already gave.
2001    fn landing_base(&self) -> String {
2002        self.state
2003            .base_sync
2004            .as_ref()
2005            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
2006    }
2007
2008    // --------------------------------------------------------------- review
2009
2010    async fn review_loop(&mut self) -> Result<()> {
2011        // A base that would not rebase is a person's decision, not a review
2012        // round: nothing here would change the answer, and reviewers and a
2013        // fixer would be spending real budget on a tree that cannot land
2014        // regardless of what they find.
2015        if self
2016            .state
2017            .base_sync
2018            .as_ref()
2019            .is_some_and(|s| s.conflict.is_some())
2020        {
2021            return Ok(());
2022        }
2023        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2024        // agent files with `magi task add` name the run that paid for it. The
2025        // prompt overlay is cloned alongside it because the waves borrow it
2026        // while `self` is mutably borrowed by the node's own bookkeeping.
2027        let run_id = self.state.id.clone();
2028        let prompts = self.state.config.prompts.clone();
2029        let Some(winner) = self.state.winner().cloned() else {
2030            return Ok(());
2031        };
2032        let max_rounds = self.state.config.graph.review_rounds;
2033        if max_rounds == 0 || self.state.reviews.iter().any(|r| r.clean) {
2034            self.state.status = RunStatus::Gating;
2035            self.state.save()?;
2036            return Ok(());
2037        }
2038        if self.state.reviews.len() >= max_rounds {
2039            // Every round is already spent and none was clean — this is a
2040            // reentry into a run that reached `Blocked` and stopped. The loop
2041            // below runs an empty range in that case and would otherwise fall
2042            // through without touching `status`, silently handing back
2043            // whatever an earlier node in this same walk clobbered it to (a
2044            // solo-candidate `judge`/`deliberate` skip both rewrite `status`
2045            // on every reentry). Restate the conclusion instead of leaving it
2046            // to chance.
2047            self.state.status = RunStatus::Blocked;
2048            self.state.save()?;
2049            return Ok(());
2050        }
2051        self.state.status = RunStatus::Reviewing;
2052
2053        let repo = self.state.repo.clone();
2054        let root = self.state.worktree_root();
2055        let language = self.state.config.graph.language.clone();
2056        let sessions = self.state.config.graph.sessions;
2057        let artifacts = agent::artifacts_dir(&self.state.dir());
2058        let base = self.landing_base();
2059        let base_short = short(&base);
2060        let reviewers = self.roles.reviewers.clone();
2061        let shell = self.state.config.shell();
2062
2063        let mut prev_e2e: Option<String> = None;
2064        for round in (self.state.reviews.len() + 1)..=max_rounds {
2065            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
2066            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
2067            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
2068
2069            // Each reviewer gets its own detached checkout of exactly this
2070            // commit: nobody can perturb the winner's tree, and the fixer can
2071            // keep working without racing a reviewer.
2072            let mut jobs = Vec::new();
2073            for (r, spec) in reviewers.iter().cloned().enumerate() {
2074                let wt = root.join(format!("review-{}", r + 1));
2075                if wt.exists() {
2076                    git::reset_detached(&wt, &head).await?;
2077                } else {
2078                    git::worktree_add_detached(&repo, &wt, &head).await?;
2079                }
2080                let seat_key = format!("review-{}", r + 1);
2081                let seat = self.seat(&seat_key, &spec.id);
2082                jobs.push(SeatJob {
2083                    prompt: prompt::review(&prompt::ReviewCtx {
2084                        instruction: &self.state.instruction,
2085                        branch: &winner.branch,
2086                        base_short: &base_short,
2087                        stat: &stat,
2088                        patch: &patch,
2089                        e2e: prev_e2e.as_deref(),
2090                        reviewers: reviewers.len(),
2091                        round,
2092                        rounds: max_rounds,
2093                        // A review-only run has no rankings, so nothing
2094                        // competed for this patch and the reviewer is told so.
2095                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
2096                        language: &language,
2097                    }),
2098                    spec,
2099                    seat,
2100                    cwd: wt,
2101                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
2102                    allow_write: false,
2103                    sessions,
2104                    artifacts: artifacts.clone(),
2105                    stem: format!("review-{round}-{}", r + 1),
2106                });
2107            }
2108
2109            self.state.event(
2110                "review",
2111                format!(
2112                    "round {round}: {} reviewers on {}",
2113                    jobs.len(),
2114                    short(&head)
2115                ),
2116            );
2117            let mut quota_losses = Vec::new();
2118            let review_retries = self.state.config.graph.retries;
2119            let review_cache = self.state.config.cache_dir();
2120            let ctx = WaveCtx {
2121                run: &run_id,
2122                node: "review",
2123                prompts: &prompts,
2124                cache: review_cache.as_deref(),
2125            };
2126            let results = ask_json_wave::<Review>(
2127                jobs,
2128                Arc::clone(&self.sem),
2129                review_retries,
2130                &ctx,
2131                &mut quota_losses,
2132                &mut self.state,
2133                &|_: &Review| Ok(()),
2134            )
2135            .await;
2136            self.state.quota.extend(quota_losses);
2137
2138            let mut records = Vec::new();
2139            let mut all_findings = Vec::new();
2140            for (r, (seat, res)) in results.into_iter().enumerate() {
2141                let agent_id = seat.agent.clone();
2142                self.state.seats.insert(seat.key.clone(), seat);
2143                let mut record = ReviewRecord {
2144                    reviewer: r + 1,
2145                    agent: agent_id,
2146                    summary: String::new(),
2147                    findings: Vec::new(),
2148                    failed: None,
2149                    duration_ms: 0,
2150                };
2151                match res {
2152                    Ok((review, out)) => {
2153                        record.summary = review.summary;
2154                        record.duration_ms = out.duration_ms;
2155                        for (n, mut f) in review.findings.into_iter().enumerate() {
2156                            // ids are magi's, never the agent's: the fixer's
2157                            // adoption report is keyed by them.
2158                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
2159                            all_findings.push(f.clone());
2160                            record.findings.push(f);
2161                        }
2162                        self.state.event(
2163                            "review",
2164                            format!(
2165                                "round {round}: reviewer {} raised {} finding(s)",
2166                                r + 1,
2167                                record.findings.len()
2168                            ),
2169                        );
2170                    }
2171                    Err(e) => {
2172                        record.failed = Some(e.to_string());
2173                        self.state.event(
2174                            "review",
2175                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
2176                        );
2177                    }
2178                }
2179                records.push(record);
2180            }
2181
2182            let mut e2e = run_commands(
2183                &shell,
2184                &self.state.config.verify.e2e,
2185                &winner.worktree,
2186                Duration::from_secs(self.state.config.graph.timeout_review),
2187            )
2188            .await;
2189            for o in &e2e {
2190                self.state.event(
2191                    "verify",
2192                    format!("round {round}: `{}` -> {}", o.command, e2e_outcome_label(o)),
2193                );
2194            }
2195
2196            // A build/link failure is not a verdict on the patch — it is
2197            // frequently a race against a shared `CARGO_TARGET_DIR` (see
2198            // AGENTS.md). Give verify one retry before letting a red like
2199            // that decide the round.
2200            let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
2201            if verify_retried {
2202                self.state.event(
2203                    "verify",
2204                    format!(
2205                        "round {round}: verify could not build/link, not a test result — \
2206                         retrying once before concluding"
2207                    ),
2208                );
2209                e2e = run_commands(
2210                    &shell,
2211                    &self.state.config.verify.e2e,
2212                    &winner.worktree,
2213                    Duration::from_secs(self.state.config.graph.timeout_review),
2214                )
2215                .await;
2216                for o in &e2e {
2217                    self.state.event(
2218                        "verify",
2219                        format!(
2220                            "round {round}: retry `{}` -> {}",
2221                            o.command,
2222                            e2e_outcome_label(o)
2223                        ),
2224                    );
2225                }
2226            }
2227
2228            let e2e_failures: String = e2e
2229                .iter()
2230                .filter(|o| !o.ok())
2231                .map(|o| format!("$ {}\n{}\n", o.command, o.output_tail))
2232                .collect();
2233
2234            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
2235            let clean = blocking == 0 && e2e.iter().all(CommandOutcome::ok);
2236
2237            let mut round_record = ReviewRound {
2238                round,
2239                head: head.clone(),
2240                reviews: records,
2241                e2e,
2242                verify_retried,
2243                fix: None,
2244                blocking,
2245                clean,
2246            };
2247
2248            if clean {
2249                self.state.event(
2250                    "review",
2251                    format!("round {round}: clean — no blocking findings, verification green"),
2252                );
2253                self.state.reviews.push(round_record);
2254                self.state.status = RunStatus::Gating;
2255                self.state.save()?;
2256                return Ok(());
2257            }
2258
2259            if round == max_rounds {
2260                self.state.reviews.push(round_record);
2261                self.state.status = RunStatus::Blocked;
2262                self.state.event(
2263                    "review",
2264                    format!("{blocking} blocking finding(s) still open after {max_rounds} rounds"),
2265                );
2266                self.state.save()?;
2267                return Ok(());
2268            }
2269
2270            // Fix. The winner's own implementer seat continues its conversation:
2271            // the competition is over, so context is pure benefit now.
2272            let (fix_spec, fix_seat_key) = match &self.roles.fixer {
2273                Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
2274                _ => (
2275                    self.state
2276                        .config
2277                        .agent(&winner.agent)
2278                        .cloned()
2279                        .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
2280                    format!("impl-{}", winner.label),
2281                ),
2282            };
2283            let seat = self.seat(&fix_seat_key, &fix_spec.id);
2284            let blocking_findings: Vec<_> = all_findings
2285                .iter()
2286                .filter(|f| f.severity.blocks())
2287                .cloned()
2288                .collect();
2289            let job = SeatJob {
2290                prompt: prompt::fix(
2291                    &self.state.instruction,
2292                    &blocking_findings,
2293                    (!e2e_failures.is_empty()).then_some(e2e_failures.as_str()),
2294                    round,
2295                    max_rounds,
2296                    &language,
2297                ),
2298                spec: fix_spec.clone(),
2299                seat,
2300                cwd: winner.worktree.clone(),
2301                timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
2302                allow_write: true,
2303                sessions,
2304                artifacts: artifacts.clone(),
2305                stem: format!("fix-{round}"),
2306            };
2307            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
2308            let cache = self.state.config.cache_dir();
2309            let ctx = WaveCtx {
2310                run: &run_id,
2311                node: "fix",
2312                prompts: &prompts,
2313                cache: cache.as_deref(),
2314            };
2315            let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2316            let agent_id = seat.agent.clone();
2317            let seat_key = seat.key.clone();
2318            self.state.seats.insert(seat.key.clone(), seat);
2319
2320            let mut fix = FixRecord {
2321                agent: agent_id,
2322                addressed: Vec::new(),
2323                rejected: Vec::new(),
2324                notes: String::new(),
2325                committed: false,
2326                failed: None,
2327                duration_ms: 0,
2328            };
2329            match out {
2330                AgentOutcome::Ok(o) => {
2331                    fix.duration_ms = o.duration_ms;
2332                    match verdict::extract_json::<FixReport>(&o.text) {
2333                        Ok(report) => {
2334                            fix.addressed = report.addressed;
2335                            fix.rejected = report.rejected;
2336                            fix.notes =
2337                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
2338                        }
2339                        Err(e) => fix.failed = Some(format!("unparsable fix report: {e}")),
2340                    }
2341                }
2342                // The CLI's raw error JSON is not a fix report to parse.
2343                AgentOutcome::Dropped(o) => {
2344                    fix.duration_ms = o.duration_ms;
2345                    let why = o
2346                        .dropped
2347                        .as_ref()
2348                        .map(|d| d.why.as_str())
2349                        .unwrap_or("the CLI ended the stream without delivering its answer");
2350                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
2351                }
2352                AgentOutcome::Quota(o) => {
2353                    self.state.quota.push(QuotaLoss {
2354                        seat: seat_key,
2355                        node: "fix".to_owned(),
2356                        at: Timestamp::now(),
2357                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2358                    });
2359                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
2360                }
2361                AgentOutcome::Failed(e) => fix.failed = Some(e),
2362            }
2363            git::commit_all(
2364                &winner.worktree,
2365                &format!("magi: review round {round} fixes (uncommitted work)"),
2366            )
2367            .await
2368            .ok();
2369            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
2370            fix.committed = after != before;
2371            let commit_note = if fix.committed {
2372                "committed"
2373            } else {
2374                "NO new commit"
2375            };
2376            self.state.event(
2377                "fix",
2378                match &fix.failed {
2379                    // Distinct on purpose from "0 addressed, 0 rejected": the
2380                    // fixer's own diff still landed (blocking counts do keep
2381                    // falling round over round), only its adoption report did
2382                    // not come back, so this must never read like every
2383                    // finding was reviewed and declined.
2384                    Some(reason) => {
2385                        format!("round {round}: fixer's adoption report was lost ({reason}); {commit_note}")
2386                    }
2387                    None => format!(
2388                        "round {round}: {} addressed, {} rejected, {commit_note}",
2389                        fix.addressed.len(),
2390                        fix.rejected.len(),
2391                    ),
2392                },
2393            );
2394            let stalled = !fix.committed;
2395            round_record.fix = Some(fix);
2396            self.state.reviews.push(round_record);
2397            self.state.save()?;
2398
2399            prev_e2e = (!e2e_failures.is_empty()).then_some(e2e_failures);
2400
2401            if stalled {
2402                self.state.status = RunStatus::Blocked;
2403                self.state.event(
2404                    "review",
2405                    "the fixer produced no commit; stopping instead of looping on an unchanged tree"
2406                        .to_owned(),
2407                );
2408                self.state.save()?;
2409                return Ok(());
2410            }
2411        }
2412        Ok(())
2413    }
2414
2415    // ----------------------------------------------------------------- gate
2416
2417    async fn gate(&mut self) -> Result<()> {
2418        // Judged by the review record itself, not by `status`: a solo
2419        // candidate's `judge`/`deliberate` skip rewrites `status` on every
2420        // reentry (see `judge`), and trusting it here is exactly how a run
2421        // that exhausted its review budget got gated and merged a second
2422        // time around. The last round's `clean` flag is the actual verdict.
2423        // A base the winner could not be replayed onto is a decision, not a
2424        // round: there is no landing tree to gate. Read as its own record for
2425        // the same reason the review verdict is.
2426        if self.state.status == RunStatus::Failed
2427            || self
2428                .state
2429                .base_sync
2430                .as_ref()
2431                .is_some_and(|s| s.conflict.is_some())
2432            || self.state.reviews.last().is_some_and(|r| !r.clean)
2433        {
2434            return Ok(());
2435        }
2436        if !self.state.gate.is_empty() {
2437            return Ok(());
2438        }
2439        let Some(winner) = self.state.winner().cloned() else {
2440            return Ok(());
2441        };
2442        self.state.status = RunStatus::Gating;
2443        let shell = self.state.config.shell();
2444        let outcomes = run_commands(
2445            &shell,
2446            &self.state.config.verify.gate,
2447            &winner.worktree,
2448            Duration::from_secs(self.state.config.graph.timeout_review),
2449        )
2450        .await;
2451        for o in &outcomes {
2452            self.state.event(
2453                "gate",
2454                format!(
2455                    "`{}` -> {}",
2456                    o.command,
2457                    if o.ok() {
2458                        "pass".to_owned()
2459                    } else {
2460                        format!("FAIL ({:?})", o.code)
2461                    }
2462                ),
2463            );
2464        }
2465        let passed = outcomes.iter().all(CommandOutcome::ok);
2466        self.state.gate = outcomes;
2467        if !passed {
2468            self.state.status = RunStatus::Blocked;
2469            self.state.event("gate", "gate failed; not merging");
2470        }
2471        self.state.save()?;
2472        Ok(())
2473    }
2474
2475    // ---------------------------------------------------------------- merge
2476
2477    async fn merge(&mut self) -> Result<()> {
2478        // Same reasoning as `gate`: ask the review and gate records directly
2479        // rather than `status`, which a solo-candidate `judge`/`deliberate`
2480        // skip can rewrite on reentry to something that no longer says
2481        // `Blocked`.
2482        if self
2483            .state
2484            .base_sync
2485            .as_ref()
2486            .is_some_and(|s| s.conflict.is_some())
2487            || self.state.reviews.last().is_some_and(|r| !r.clean)
2488            || self.state.gate.iter().any(|o| !o.ok())
2489        {
2490            return Ok(());
2491        }
2492        // This node's own record, not `status`: `status == Ready` is not
2493        // unique to the harmless `MergeMode::None` path this line was
2494        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
2495        // run's PR was closed without merging — and on that run `mode` is
2496        // still `Pr`, so a reentry that fell through here would push and
2497        // open a second pull request. `self.state.merge` is set exactly once
2498        // this node (or `land`) has already produced a verdict, under every
2499        // mode, which is what "already done" actually means here.
2500        if self.state.merge.is_some() {
2501            return Ok(());
2502        }
2503        let Some(winner) = self.state.winner().cloned() else {
2504            return Ok(());
2505        };
2506        let repo = self.state.repo.clone();
2507        let base = self.state.base_branch.clone();
2508        let mode = self.state.config.merge.mode;
2509        let message = format!(
2510            "Merge magi run {} (candidate {})\n\n{}",
2511            self.state.id, winner.label, self.state.instruction
2512        );
2513
2514        let outcome = match mode {
2515            MergeMode::None => MergeOutcome {
2516                mode,
2517                ok: true,
2518                detail: format!("git -C {} merge --no-ff {}", repo.display(), winner.branch),
2519            },
2520            MergeMode::Local => {
2521                let on = git::current_branch(&repo).await?;
2522                if on.as_deref() != Some(base.as_str()) {
2523                    MergeOutcome {
2524                        mode,
2525                        ok: false,
2526                        detail: format!(
2527                            "{} has {} checked out, not the base branch {base}",
2528                            repo.display(),
2529                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
2530                        ),
2531                    }
2532                } else if !git::is_clean(&repo).await? {
2533                    MergeOutcome {
2534                        mode,
2535                        ok: false,
2536                        detail: format!("{} is dirty; refusing to merge", repo.display()),
2537                    }
2538                } else {
2539                    let out = git::merge_no_ff(&repo, &winner.branch, &message).await?;
2540                    MergeOutcome {
2541                        mode,
2542                        ok: out.ok(),
2543                        detail: if out.ok() { out.stdout } else { out.stderr },
2544                    }
2545                }
2546            }
2547            MergeMode::Pr => {
2548                let remote = self.state.config.merge.remote.clone();
2549                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
2550                if !pushed.ok() {
2551                    MergeOutcome {
2552                        mode,
2553                        ok: false,
2554                        detail: pushed.stderr,
2555                    }
2556                } else {
2557                    let out = gh_pr_create(&winner.worktree, &base, &winner.branch, &message).await;
2558                    match out {
2559                        Ok(url) => MergeOutcome {
2560                            mode,
2561                            ok: true,
2562                            detail: url,
2563                        },
2564                        Err(e) => MergeOutcome {
2565                            mode,
2566                            ok: false,
2567                            detail: e.to_string(),
2568                        },
2569                    }
2570                }
2571            }
2572        };
2573
2574        self.state.status = match (mode, outcome.ok) {
2575            (MergeMode::None, _) => RunStatus::Ready,
2576            (_, true) => RunStatus::Merged,
2577            (_, false) => RunStatus::Blocked,
2578        };
2579        self.state.event(
2580            "merge",
2581            format!(
2582                "{:?}: {}",
2583                mode,
2584                outcome.detail.lines().next().unwrap_or("")
2585            ),
2586        );
2587        self.state.merge = Some(outcome);
2588        self.state.save()?;
2589
2590        // The PR is open and the run would historically stop here, leaving the
2591        // operator to watch checks, feed review comments back to a fixer, and
2592        // merge. That was done by hand six times in one session before this
2593        // existed. Opt-in, because merging is the one irreversible thing magi
2594        // can do to a repository.
2595        if self.state.config.graph.land
2596            && mode == MergeMode::Pr
2597            && self.state.status == RunStatus::Merged
2598        {
2599            let url = self
2600                .state
2601                .merge
2602                .as_ref()
2603                .map(|m| m.detail.clone())
2604                .unwrap_or_default();
2605            let url = url.lines().next().unwrap_or("").trim().to_owned();
2606            if url.starts_with("http") {
2607                // A land failure is not a lost run: the work is on a branch and
2608                // the PR is open, which is exactly where a human takes over.
2609                match land::land(&mut self.state, &url).await {
2610                    Ok(pr) => {
2611                        self.state.status = match pr.state {
2612                            land::PrLifecycle::Merged => RunStatus::Merged,
2613                            _ => RunStatus::Blocked,
2614                        };
2615                    }
2616                    Err(e) => {
2617                        self.state.status = RunStatus::Blocked;
2618                        self.state.event("land", format!("gave up: {e}"));
2619                    }
2620                }
2621                self.state.save()?;
2622            }
2623        }
2624        Ok(())
2625    }
2626
2627    // -------------------------------------------------------------- helpers
2628
2629    /// Fetch or create a seat, keeping its conversation across nodes.
2630    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
2631        if let Some(existing) = self.state.seats.get(key)
2632            && existing.agent == agent
2633        {
2634            return existing.clone();
2635        }
2636        let fresh = SeatState::new(key, agent, self.state.seed);
2637        self.state.seats.insert(key.to_owned(), fresh.clone());
2638        fresh
2639    }
2640
2641    /// A candidate rendered for judging, with the leak policy applied.
2642    fn view(&self, c: &Candidate) -> CandidateView {
2643        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2644            .unwrap_or_default();
2645        let (patch, _) = blind::sanitize_patch(
2646            &format!("candidate {} patch", c.label),
2647            &raw,
2648            &self.state.config.blind,
2649        );
2650        CandidateView {
2651            label: c.label,
2652            branch: c.branch.clone(),
2653            summary: c.summary.clone(),
2654            stat: c.stat.clone(),
2655            patch,
2656        }
2657    }
2658
2659    /// The full candidate set as prompt text, for seats with no live session.
2660    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
2661        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
2662        prompt::judge(
2663            "(see above)",
2664            &views,
2665            self.roles.judges.len(),
2666            base_short,
2667            "en",
2668        )
2669    }
2670
2671    /// Anonymised transcript for judge `self_idx`.
2672    ///
2673    /// The initial rankings are always the opening statements. Seeding them
2674    /// only when no turn had been taken yet meant every judge after the first
2675    /// argued against a single voice instead of against the actual split — the
2676    /// disagreement is the information, so it is always on the table.
2677    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
2678        let mut turns = Vec::new();
2679        for j in &self.state.judgements {
2680            if j.ranking.is_empty() {
2681                continue;
2682            }
2683            let reasons = j
2684                .reasons
2685                .iter()
2686                .map(|(k, v)| format!("- {k}: {v}"))
2687                .collect::<Vec<_>>()
2688                .join("\n");
2689            turns.push(Turn {
2690                who: format!("Judge {} (opening ranking)", j.judge),
2691                is_self: j.judge == self_idx + 1,
2692                body: format!(
2693                    "Ranked {}{}{reasons}",
2694                    j.ranking.iter().collect::<String>(),
2695                    if reasons.is_empty() {
2696                        ""
2697                    } else {
2698                        ", because:\n"
2699                    }
2700                ),
2701            });
2702        }
2703        for t in self
2704            .state
2705            .deliberation
2706            .iter()
2707            .flat_map(|r| r.turns.iter())
2708            .chain(current)
2709        {
2710            turns.push(Turn {
2711                who: format!("Judge {}", t.judge),
2712                is_self: t.judge == self_idx + 1,
2713                body: t.body.clone(),
2714            });
2715        }
2716        turns
2717    }
2718}
2719
2720/// Does this seat still hold the context a follow-up prompt would rely on?
2721fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
2722    agent::has_session(spec.kind, seat, sessions)
2723}
2724
2725fn short(commit: &str) -> String {
2726    commit.chars().take(7).collect()
2727}
2728
2729fn make_executable(path: &Path) -> Result<()> {
2730    #[cfg(unix)]
2731    {
2732        use std::os::unix::fs::PermissionsExt as _;
2733        let mut perms = std::fs::metadata(path)?.permissions();
2734        perms.set_mode(0o755);
2735        std::fs::set_permissions(path, perms)?;
2736    }
2737    #[cfg(not(unix))]
2738    {
2739        let _ = path;
2740    }
2741    Ok(())
2742}
2743
2744/// What every seat in one batch shares: where the answers are attributed, the
2745/// prompt overlay they inherit, and the build cache they are told to use.
2746///
2747/// A struct rather than four more parameters: `wave` also needs the run's
2748/// state (to record who is answering right now) and the attempt number, and
2749/// eight positional arguments is both unreadable and a clippy error.
2750struct WaveCtx<'a> {
2751    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
2752    /// paid for it.
2753    run: &'a str,
2754    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
2755    node: &'a str,
2756    prompts: &'a Prompts,
2757    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
2758    cache: Option<&'a Path>,
2759}
2760
2761/// Run one job, honouring the parallelism budget.
2762async fn run_one(
2763    job: SeatJob,
2764    sem: Arc<Semaphore>,
2765    ctx: &WaveCtx<'_>,
2766    state: &mut RunState,
2767    attempt: usize,
2768) -> (SeatState, AgentOutcome) {
2769    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
2770        .await
2771        .pop()
2772        .expect("one job in, one result out");
2773    (seat, out)
2774}
2775
2776/// Run every job concurrently, capped by the semaphore, preserving order.
2777///
2778/// Every seat in the batch is recorded into [`RunState::active`] before the
2779/// wave starts and cleared as each answer lands, so the run's own record says
2780/// who is still being waited on rather than only who finished.
2781async fn wave(
2782    jobs: Vec<SeatJob>,
2783    sem: Arc<Semaphore>,
2784    ctx: &WaveCtx<'_>,
2785    state: &mut RunState,
2786    attempt: usize,
2787) -> Vec<(usize, SeatState, AgentOutcome)> {
2788    let WaveCtx {
2789        run,
2790        node,
2791        prompts,
2792        cache,
2793    } = *ctx;
2794    for job in &jobs {
2795        state.seat_started(node, &job.seat.key, job.timeout, attempt);
2796    }
2797    if let Err(e) = state.save() {
2798        // A failed persist of "who is answering right now" must not abort the
2799        // wave: the seats are already being asked, and the alternative is
2800        // losing the answers to save a status line nobody may even be
2801        // watching.
2802        tracing::warn!("could not persist in-progress seats: {e:#}");
2803    }
2804    let mut set = tokio::task::JoinSet::new();
2805    let overlay = prompts.overlay(node);
2806    for (i, mut job) in jobs.into_iter().enumerate() {
2807        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
2808        if cache.is_some() {
2809            job.prompt.push('\n');
2810            job.prompt.push_str(prompt::build_cache_note());
2811        }
2812        let sem = Arc::clone(&sem);
2813        let run = run.to_owned();
2814        let node = node.to_owned();
2815        let cache = cache.map(Path::to_path_buf);
2816        set.spawn(async move {
2817            let _permit = sem.acquire().await;
2818            let mut seat = job.seat;
2819            let out = agent::invoke(
2820                &job.spec,
2821                &mut seat,
2822                &Invocation {
2823                    cwd: &job.cwd,
2824                    prompt: &job.prompt,
2825                    timeout: job.timeout,
2826                    allow_write: job.allow_write,
2827                    sessions: job.sessions,
2828                    artifacts: &job.artifacts,
2829                    stem: &job.stem,
2830                    run: &run,
2831                    node: &node,
2832                    cache_dir: cache.as_deref(),
2833                },
2834            )
2835            .await;
2836            let out = match out {
2837                Ok(o) if o.usable() => AgentOutcome::Ok(o),
2838                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
2839                // Billed work the CLI failed to hand over is not an ordinary
2840                // failure, but its text is the CLI's raw error JSON, not an
2841                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
2842                // read it as one by forgetting to check. `usable()` is always
2843                // false here (dropped implies an empty response), so this has
2844                // to be checked before the catch-all `Failed` below or the
2845                // one shape this exists for is lost with the rest.
2846                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
2847                Ok(o) if o.timed_out => AgentOutcome::Failed("timed out".to_owned()),
2848                Ok(o) => AgentOutcome::Failed(format!(
2849                    "exited with {:?} and no usable output",
2850                    o.exit_code
2851                )),
2852                Err(e) => AgentOutcome::Failed(e.to_string()),
2853            };
2854            (i, seat, out)
2855        });
2856    }
2857    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
2858    while let Some(joined) = set.join_next().await {
2859        let (i, seat, out) = match joined {
2860            Ok(v) => v,
2861            // No seat to clear: a panicked task never reported which one it
2862            // was. The defensive sweep below this loop is what stops that
2863            // seat's `active` entry from surviving forever.
2864            Err(e) => {
2865                tracing::error!("agent task panicked: {e}");
2866                continue;
2867            }
2868        };
2869        state.seat_finished(&seat.key);
2870        if let Err(e) = state.save() {
2871            tracing::warn!("could not persist a seat's completion: {e:#}");
2872        }
2873        if collected.len() <= i {
2874            collected.resize_with(i + 1, || None);
2875        }
2876        collected[i] = Some((i, seat, out));
2877    }
2878    // Belt-and-braces for the panic branch above: every seat this exact batch
2879    // started shares this `(node, attempt)` pair, and every seat that finished
2880    // normally already cleared itself, so anything left tagged with it here
2881    // can only be a panicked task's leftover. Cleared unconditionally rather
2882    // than left to read as still answering forever.
2883    if state
2884        .active
2885        .values()
2886        .any(|a| a.node == node && a.attempt == attempt)
2887    {
2888        state
2889            .active
2890            .retain(|_, a| !(a.node == node && a.attempt == attempt));
2891        if let Err(e) = state.save() {
2892            tracing::warn!("could not persist the end of a wave: {e:#}");
2893        }
2894    }
2895    collected.into_iter().flatten().collect()
2896}
2897
2898/// How long a re-ask may take, given the budget the first attempt had.
2899///
2900/// A `nudged` retry is a request to restate an answer the seat has already
2901/// worked out: it carries no new work, so it does not deserve the original
2902/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
2903/// 133 seconds while a third sat for over ten minutes on a resumed session
2904/// holding 410 KB of prior output - and because the retry had inherited the
2905/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
2906/// judging round whose other seats were long finished.
2907///
2908/// A quarter of the budget, with a floor so that a deliberately short timeout
2909/// does not collapse to nothing. A retry that re-sends the whole prompt
2910/// (because the seat kept no context) is the original job again, and keeps the
2911/// original budget.
2912fn retry_budget(full: Duration, nudged: bool) -> Duration {
2913    if nudged {
2914        (full / 4).max(Duration::from_secs(120)).min(full)
2915    } else {
2916        full
2917    }
2918}
2919
2920/// Run a wave and parse each reply, re-asking the seats whose reply was
2921/// unusable.
2922///
2923/// The re-ask is a nudge rather than the whole prompt again when the seat still
2924/// holds its conversation, which is the difference between a cheap retry and
2925/// paying for the entire candidate set twice.
2926///
2927/// A seat that hits a rate limit is **not** re-asked: the same call will fail
2928/// the same way until the limit resets, so spending a retry attempt on it is
2929/// pure waste. Its loss is recorded in `losses` and it is returned as a failure
2930/// like any other absent seat — the caller decides whether the panel still has
2931/// a quorum.
2932#[allow(clippy::too_many_arguments)]
2933async fn ask_json_wave<T>(
2934    jobs: Vec<SeatJob>,
2935    sem: Arc<Semaphore>,
2936    retries: usize,
2937    ctx: &WaveCtx<'_>,
2938    losses: &mut Vec<QuotaLoss>,
2939    state: &mut RunState,
2940    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
2941) -> Vec<(SeatState, Result<(T, AgentOutput)>)>
2942where
2943    T: serde::de::DeserializeOwned + Send + 'static,
2944{
2945    let n = jobs.len();
2946    let originals: Vec<SeatJob> = jobs;
2947    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
2948    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
2949    let mut pending: Vec<usize> = (0..n).collect();
2950
2951    for attempt in 0..=retries {
2952        if pending.is_empty() {
2953            break;
2954        }
2955        let mut batch = Vec::with_capacity(pending.len());
2956        for &i in &pending {
2957            let src = &originals[i];
2958            // The prompt and the budget are one decision: a nudge restates
2959            // finished work, a re-sent prompt redoes it.
2960            let (prompt, timeout) = if attempt == 0 {
2961                (src.prompt.clone(), src.timeout)
2962            } else {
2963                let why = done[i]
2964                    .as_ref()
2965                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
2966                    .unwrap_or_else(|| "no parsable answer".to_owned());
2967                let nudge = prompt::nudge(&why);
2968                let nudged = has_context(&src.spec, &seats[i], src.sessions);
2969                let prompt = if nudged {
2970                    nudge
2971                } else {
2972                    format!("{}\n\n---\n\n{}", src.prompt, nudge)
2973                };
2974                (prompt, retry_budget(src.timeout, nudged))
2975            };
2976            batch.push(SeatJob {
2977                spec: src.spec.clone(),
2978                seat: seats[i].clone(),
2979                cwd: src.cwd.clone(),
2980                prompt,
2981                timeout,
2982                allow_write: src.allow_write,
2983                sessions: src.sessions,
2984                artifacts: src.artifacts.clone(),
2985                stem: if attempt == 0 {
2986                    src.stem.clone()
2987                } else {
2988                    format!("{}-retry{attempt}", src.stem)
2989                },
2990            });
2991        }
2992
2993        if attempt > 0 {
2994            let seats_out: Vec<&str> = pending
2995                .iter()
2996                .map(|&i| originals[i].seat.key.as_str())
2997                .collect();
2998            state.event(
2999                ctx.node,
3000                format!("retry {attempt}: re-asking {}", seats_out.join(", ")),
3001            );
3002        }
3003        let results = wave(batch, Arc::clone(&sem), ctx, state, attempt).await;
3004        let mut still = Vec::new();
3005        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
3006            seats[i] = seat;
3007            let (parsed, quota) = match out {
3008                AgentOutcome::Ok(o) => (
3009                    match verdict::extract_json::<T>(&o.text) {
3010                        Ok(v) => match validate(&v) {
3011                            Ok(()) => Ok((v, o)),
3012                            Err(e) => Err(e),
3013                        },
3014                        Err(e) => Err(e),
3015                    },
3016                    false,
3017                ),
3018                AgentOutcome::Quota(o) => {
3019                    losses.push(QuotaLoss {
3020                        seat: originals[i].seat.key.clone(),
3021                        node: ctx.node.to_owned(),
3022                        at: Timestamp::now(),
3023                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3024                    });
3025                    (
3026                        Err(anyhow::anyhow!("rate limited (quota); not retrying now")),
3027                        true,
3028                    )
3029                }
3030                // Not a parseable answer, but also not worth a special-cased
3031                // retry here: the nudge loop above already re-asks anything
3032                // that fails to parse, which is exactly what a dropped stream
3033                // needs. Just don't hand its raw error JSON to `extract_json`.
3034                AgentOutcome::Dropped(o) => {
3035                    let why = o
3036                        .dropped
3037                        .as_ref()
3038                        .map(|d| d.why.as_str())
3039                        .unwrap_or("the CLI ended the stream without delivering its answer");
3040                    (
3041                        Err(anyhow::anyhow!("the CLI dropped the stream ({why})")),
3042                        false,
3043                    )
3044                }
3045                AgentOutcome::Failed(e) => (Err(anyhow::anyhow!(e)), false),
3046            };
3047            let failed = parsed.is_err();
3048            done[i] = Some(parsed);
3049            // Do not re-ask a rate-limited seat (quota) — a retry is known to
3050            // fail the same way; and never re-ask a seat that already parsed.
3051            if failed && !quota {
3052                still.push(i);
3053            }
3054        }
3055        pending = still;
3056    }
3057
3058    seats
3059        .into_iter()
3060        .zip(done)
3061        .map(|(seat, res)| {
3062            (
3063                seat,
3064                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
3065            )
3066        })
3067        .collect()
3068}
3069
3070/// Describe one verify command's outcome for the event log, distinguishing a
3071/// build/link failure — the toolchain never produced a binary to run — from
3072/// an actual test failure, since only the latter is a verdict on the patch.
3073fn e2e_outcome_label(o: &CommandOutcome) -> String {
3074    if o.ok() {
3075        "pass".to_owned()
3076    } else if o.build_failed() {
3077        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
3078    } else {
3079        format!("FAIL ({:?})", o.code)
3080    }
3081}
3082
3083/// Run configured shell commands in `cwd`, in order.
3084async fn run_commands(
3085    shell: &[String],
3086    commands: &[String],
3087    cwd: &Path,
3088    timeout: Duration,
3089) -> Vec<CommandOutcome> {
3090    let mut out = Vec::new();
3091    for command in commands {
3092        let started = Instant::now();
3093        let mut cmd = tokio::process::Command::new(&shell[0]);
3094        cmd.quiet();
3095        cmd.args(&shell[1..])
3096            .arg(command)
3097            .current_dir(cwd)
3098            .stdin(std::process::Stdio::null())
3099            .stdout(std::process::Stdio::piped())
3100            .stderr(std::process::Stdio::piped())
3101            .kill_on_drop(true);
3102        let spawned = cmd.spawn();
3103        let (code, body) = match spawned {
3104            Ok(child) => match tokio::time::timeout(timeout, child.wait_with_output()).await {
3105                Ok(Ok(o)) => {
3106                    let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
3107                    body.push_str(&String::from_utf8_lossy(&o.stderr));
3108                    (o.status.code(), body)
3109                }
3110                Ok(Err(e)) => (None, format!("failed to run: {e}")),
3111                Err(_) => (None, format!("timed out after {}s", timeout.as_secs())),
3112            },
3113            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
3114        };
3115        out.push(CommandOutcome {
3116            command: command.clone(),
3117            code,
3118            output_tail: tail(&body, OUTPUT_TAIL),
3119            duration_ms: started.elapsed().as_millis() as u64,
3120        });
3121    }
3122    out
3123}
3124
3125/// `gh pr create`, returning the PR url.
3126async fn gh_pr_create(cwd: &Path, base: &str, head: &str, body: &str) -> Result<String> {
3127    let title = body.lines().next().unwrap_or("magi run").to_owned();
3128    let out = tokio::process::Command::new("gh")
3129        .args([
3130            "pr", "create", "--base", base, "--head", head, "--title", &title, "--body", body,
3131        ])
3132        .current_dir(cwd)
3133        .stdin(std::process::Stdio::null())
3134        .output()
3135        .await
3136        .context("spawn gh")?;
3137    if out.status.success() {
3138        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
3139    } else {
3140        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
3141    }
3142}
3143
3144/// Tear a run's worktrees and branches down.
3145pub async fn fold_run(state: &mut RunState, drop_winner: bool) -> Result<Vec<String>> {
3146    let repo = state.repo.clone();
3147    let root = state.worktree_root();
3148    let winner = state.tally.as_ref().map(|t| t.winner);
3149    let mut removed = Vec::new();
3150
3151    for i in 0..state.candidates.len() {
3152        let c = state.candidates[i].clone();
3153        let is_winner = Some(c.label) == winner;
3154        if is_winner && !drop_winner {
3155            continue;
3156        }
3157        if c.worktree.exists() {
3158            git::worktree_remove(&repo, &c.worktree).await.ok();
3159            removed.push(c.worktree.to_string_lossy().into_owned());
3160        }
3161        if git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
3162            git::branch_delete(&repo, &c.branch).await.ok();
3163            removed.push(c.branch.clone());
3164        }
3165        state.candidates[i].folded = true;
3166    }
3167
3168    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
3169        let path = name.path();
3170        let keep = !drop_winner
3171            && winner.is_some_and(|w| {
3172                path.file_name()
3173                    .is_some_and(|n| n == format!("cand-{w}").as_str())
3174            });
3175        if keep {
3176            continue;
3177        }
3178        git::worktree_remove(&repo, &path).await.ok();
3179        removed.push(path.to_string_lossy().into_owned());
3180    }
3181
3182    if state.enabled_worktree_config && drop_winner {
3183        git::disable_worktree_config(&repo).await.ok();
3184        state.enabled_worktree_config = false;
3185    }
3186    state.save()?;
3187    Ok(removed)
3188}
3189
3190/// Severity of the worst open finding in the last review round, for reporting.
3191pub fn worst_open(state: &RunState) -> Option<Severity> {
3192    state
3193        .reviews
3194        .last()?
3195        .reviews
3196        .iter()
3197        .flat_map(|r| r.findings.iter())
3198        .map(|f| f.severity)
3199        .max()
3200}
3201
3202#[cfg(test)]
3203mod tests {
3204    use super::*;
3205    use std::time::Duration;
3206
3207    fn secs(n: u64) -> Duration {
3208        Duration::from_secs(n)
3209    }
3210
3211    /// A throwaway repo with one commit on `main`, for tests that need `merge`
3212    /// to make real (and, if it runs at all, real*ly fail*) git calls.
3213    fn init_repo(dir: &Path) {
3214        let run = |args: &[&str]| {
3215            let out = std::process::Command::new("git")
3216                .args(args)
3217                .current_dir(dir)
3218                .output()
3219                .expect("spawn git");
3220            assert!(
3221                out.status.success(),
3222                "git {args:?} failed: {}",
3223                String::from_utf8_lossy(&out.stderr)
3224            );
3225        };
3226        run(&["init", "-b", "main"]);
3227        run(&["config", "user.name", "magi test"]);
3228        run(&["config", "user.email", "magi@example.com"]);
3229        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
3230        run(&["add", "-A"]);
3231        run(&["commit", "-m", "init"]);
3232    }
3233
3234    /// `status == Ready` used to be read as "this is the harmless
3235    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
3236    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
3237    /// run's PR was closed without merging — and reentering `merge` with
3238    /// `mode` still `Pr` does not know the difference, so it pushed and
3239    /// opened a second pull request. `mode == Local` reproduces the same
3240    /// blind spot without a network call: reentry must not attempt another
3241    /// git merge once this node has already recorded an outcome.
3242    #[tokio::test]
3243    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
3244        let tmp = tempfile::tempdir().expect("tempdir");
3245        let repo = tmp.path().join("repo");
3246        std::fs::create_dir_all(&repo).unwrap();
3247        init_repo(&repo);
3248
3249        let mut config = Config::default();
3250        config.merge.mode = MergeMode::Local;
3251
3252        let mut state = RunState::new(
3253            repo.clone(),
3254            "main".to_owned(),
3255            "deadbeef".to_owned(),
3256            "task".to_owned(),
3257            config,
3258        );
3259        state.candidates = vec![Candidate {
3260            index: 0,
3261            label: 'A',
3262            agent: "alpha".to_owned(),
3263            branch: "does-not-exist".to_owned(),
3264            worktree: repo.clone(),
3265            summary: String::new(),
3266            stat: String::new(),
3267            files: 0,
3268            commits: 0,
3269            empty: false,
3270            failed: None,
3271            duration_ms: 0,
3272            folded: false,
3273        }];
3274        state.tally = Some(Tally {
3275            first_choice: BTreeMap::from([('A', 1)]),
3276            borda: BTreeMap::new(),
3277            winner: 'A',
3278            rankings: 1,
3279            unanimous_initial: true,
3280            deliberated: false,
3281            changed_votes: 0,
3282            unanimous_final: true,
3283            tie_break: None,
3284            judges: 0,
3285            present: 0,
3286            quorum: 0,
3287            met_quorum: true,
3288            uncontested: Some("only candidate A produced a change".to_owned()),
3289        });
3290        state.reviews = vec![ReviewRound {
3291            round: 1,
3292            head: "deadbeef".to_owned(),
3293            reviews: Vec::new(),
3294            e2e: Vec::new(),
3295            fix: None,
3296            blocking: 0,
3297            clean: true,
3298            verify_retried: false,
3299        }];
3300        state.gate = vec![CommandOutcome {
3301            command: "test".to_owned(),
3302            code: Some(0),
3303            output_tail: String::new(),
3304            duration_ms: 0,
3305        }];
3306        // Reached its conclusion already — e.g. `land` closing the PR without
3307        // merging it, which (like the honest `MergeMode::None` path) leaves
3308        // `status` at `Ready`. The recorded outcome is what actually marks
3309        // this node done.
3310        state.status = RunStatus::Ready;
3311        state.merge = Some(MergeOutcome {
3312            mode: MergeMode::Local,
3313            ok: false,
3314            detail: "already concluded".to_owned(),
3315        });
3316
3317        let mut runner = Runner {
3318            state,
3319            roles: ResolvedRoles {
3320                implementers: Vec::new(),
3321                judges: Vec::new(),
3322                reviewers: Vec::new(),
3323                fixer: None,
3324            },
3325            sem: Arc::new(Semaphore::new(1)),
3326            pause: Pause::new(),
3327        };
3328
3329        runner.merge().await.expect("merge");
3330
3331        assert_eq!(
3332            runner.state.status,
3333            RunStatus::Ready,
3334            "a concluded run's status must not change on reentry"
3335        );
3336        assert_eq!(
3337            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
3338            Some("already concluded"),
3339            "merge must not run again once the node already recorded an outcome"
3340        );
3341    }
3342
3343    #[test]
3344    fn a_nudge_gets_a_quarter_of_the_budget() {
3345        // The judge and implement budgets magi ships with.
3346        assert_eq!(retry_budget(secs(1200), true), secs(300));
3347        assert_eq!(retry_budget(secs(3600), true), secs(900));
3348    }
3349
3350    #[test]
3351    fn a_resent_prompt_keeps_the_whole_budget() {
3352        // The seat kept no context, so the retry is the original job again and
3353        // shortening it would only guarantee a second failure.
3354        assert_eq!(retry_budget(secs(1200), false), secs(1200));
3355        assert_eq!(retry_budget(secs(60), false), secs(60));
3356    }
3357
3358    #[test]
3359    fn the_floor_never_exceeds_the_original_budget() {
3360        // A short configured timeout must not be *raised* by the floor: the
3361        // operator asked for a bound, and a retry may not outlast the attempt
3362        // it is retrying.
3363        assert_eq!(retry_budget(secs(60), true), secs(60));
3364        assert_eq!(retry_budget(secs(480), true), secs(120));
3365        assert_eq!(retry_budget(secs(0), true), secs(0));
3366    }
3367}