Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::git;
39use crate::land;
40use crate::proc::Quiet as _;
41use crate::prompt::{
42    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
43};
44use crate::queue;
45use crate::run::{
46    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
47    DeliberationRound, DeliberationTurn, FixRecord, JobRecord, JobStatus, Judgement, MergeOutcome,
48    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, Tally,
49    VoteRecord, tail, write_artifact,
50};
51use crate::verdict::{
52    self, FinalVote, FixReport, Position, Proposal, Ranking, Review, ReviewRevote, ReviewVote,
53    Severity,
54};
55
56/// How much verification output is kept and fed back to the fixer.
57const OUTPUT_TAIL: usize = 8_000;
58
59/// Bytes of a failing command's output kept in an event, so the reason a run
60/// stopped is readable from the report without opening `run.json`.
61const EVENT_OUTPUT_TAIL: usize = 2_000;
62
63/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
64/// command's pid before releasing the build cache's lease.
65const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
66
67/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
68/// command's pid to actually exit before giving up and releasing anyway.
69///
70/// A timeout means the process was asked to die (`kill_on_drop`,
71/// `start_kill`), not that it already has — on Windows in particular that can
72/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
73/// the instant the command returns would let the very next acquirer (this
74/// run's own next round, another run's verification, the janitor's prune)
75/// start touching the same directory while it might still be writing to it,
76/// so this polls the actual pid — real confirmation, not a fixed guess —
77/// until it is gone or this ceiling is reached. It is still not full
78/// process-tree reaping: a grandchild the timed-out process spawned and that
79/// outlives it independently is invisible to a pid check, and continuing to
80/// observe and collect *that* stays a different piece of work with its own
81/// owner. Set generously because the common case returns early the moment
82/// the pid is confirmed gone, not because every timeout pays this in full.
83const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
84
85/// Consecutive review rounds with no tree progress (see
86/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
87/// instead of spending the rest of the round budget.
88///
89/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
90/// legitimately finds nothing left to change (its previous round's fix already
91/// covered it, and this round's reviewers re-raised only nits) looks the same
92/// as one that is spinning, for exactly one round. Two in a row is where the
93/// two stop being distinguishable, and a review round on this workload has
94/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
95/// third attempt at a tree that has not moved twice running is pure cost.
96/// This does not touch `review_rounds` itself, which stays the operator's
97/// call.
98pub(crate) const STAGNANT_LIMIT: usize = 2;
99
100/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
101/// a base that moved before giving up and leaving the run `Blocked` for a
102/// person.
103///
104/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
105/// that keeps moving faster than a run can catch it is not something more
106/// rebasing fixes, it is a person's call. Not the same *number as*
107/// `land_rounds` - this budget is spent before a pull request exists, land's
108/// after - but bounded for the identical reason, so it uses the same
109/// default. Counted across both call sites in [`Runner::finish_after_tally`]
110/// (once before review, once before the gate), because either one finding
111/// the base still moving is the same signal.
112const BASE_SYNC_ROUNDS: usize = 4;
113
114/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
115/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
116/// reply held no [`FixReport`].
117///
118/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
119/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
120/// "I'll pause here until the `cargo make check` background run reports
121/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
122/// No `FixReport` was ever collected from that seat, and the run moved on to
123/// the next review round regardless.
124///
125/// Bounded independently of `review_rounds` and `graph.retries`: this
126/// recovers one seat's missing report mid-round, not a new round of review or
127/// an ordinary parse retry, and must not itself become the unbounded wait the
128/// rest of this module exists to avoid.
129const MAX_FIX_CONTINUATIONS: usize = 2;
130
131/// One queued agent invocation.
132///
133/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
134/// CLI hung up on its own stream is asked again from the same job rather than
135/// rebuilt from scratch. See [`Runner::resume_undelivered`].
136#[derive(Clone)]
137struct SeatJob {
138    spec: AgentSpec,
139    seat: SeatState,
140    cwd: PathBuf,
141    prompt: String,
142    timeout: Duration,
143    allow_write: bool,
144    sessions: bool,
145    artifacts: PathBuf,
146    stem: String,
147}
148
149/// How the graph reads one agent invocation.
150///
151/// Quota is split out from an ordinary failure on purpose: a rate-limited call
152/// is known to fail again if retried now, so the retry loop must not spend an
153/// attempt on it. `Dropped` is split out for the opposite reason: unlike
154/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
155/// error JSON, never the agent's answer — a caller that matched only
156/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
157/// left to read that JSON as if it were usable output. `resume_undelivered`
158/// is the only caller that acts on it; everywhere else it is reported like an
159/// ordinary failure.
160enum AgentOutcome {
161    /// A usable output.
162    Ok(AgentOutput),
163    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
164    Quota(AgentOutput),
165    /// The CLI hung up on its own stream after billed work. See
166    /// [`agent::AgentOutput::work_undelivered`].
167    Dropped(AgentOutput),
168    /// Any other failure: a timeout, a bad exit code, an empty reply.
169    Failed(String),
170}
171
172/// A request to park the run at its next node boundary.
173///
174/// Cloning is how the request travels: the loop keeps one handle and hands a
175/// clone to each [`Runner`], and every clone points at the same flag. There
176/// is no channel because there is nothing to send - the only message is
177/// "park", it is idempotent, and a flag cannot be missed by a receiver that
178/// was not listening yet.
179///
180/// The boundary is what makes this cheap. Every node writes the run's state
181/// before the next one starts, and every node skips what is already recorded:
182/// `prep` returns early once candidates exist, `implement` asks only the seats
183/// with nothing on disk, `judge` returns early once judgements exist. So a
184/// parked run resumes into exactly the node it stopped before, and no agent
185/// work is thrown away. Killing the process mid-node, by contrast, loses
186/// whatever the seats in flight had not yet written - which for an implement
187/// wave is an hour of paid work.
188///
189/// A [`Runner`] watches two independent handles of this type - see
190/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
191/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
192/// clone covering the whole daemon's lifetime and is never asked to un-park,
193/// which is correct exactly because nothing is dispatched after it fires.
194/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
195/// that parks for an interrupted task must go on to run other tasks
196/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
197/// reusing the daemon-wide one.
198#[derive(Debug, Clone, Default)]
199pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
200
201impl Pause {
202    /// A pause nobody has asked for yet.
203    #[must_use]
204    pub fn new() -> Self {
205        Self::default()
206    }
207
208    /// Ask the run to park at its next node boundary. Idempotent.
209    pub fn park(&self) {
210        self.0.store(true, Ordering::SeqCst);
211    }
212
213    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
214    /// fold into the run's own `park` event - so an operator reading the run
215    /// later knows this was a deliberate interrupt rather than a shutdown or
216    /// a binary swap. The first reason recorded wins; a park already in
217    /// flight is not relabelled by a second, unrelated request.
218    pub fn park_because(&self, reason: impl Into<String>) {
219        let mut reason_guard = self
220            .1
221            .lock()
222            .unwrap_or_else(std::sync::PoisonError::into_inner);
223        if reason_guard.is_none() {
224            *reason_guard = Some(reason.into());
225        }
226        drop(reason_guard);
227        self.park();
228    }
229
230    /// Has a park been asked for?
231    #[must_use]
232    pub fn parked(&self) -> bool {
233        self.0.load(Ordering::SeqCst)
234    }
235
236    /// Why the park was asked for, when the caller used [`Pause::park_because`].
237    #[must_use]
238    pub fn reason(&self) -> Option<String> {
239        self.1
240            .lock()
241            .unwrap_or_else(std::sync::PoisonError::into_inner)
242            .clone()
243    }
244}
245
246/// Drives one run.
247pub struct Runner {
248    /// Run state; public so the CLI can report on it.
249    pub state: RunState,
250    roles: ResolvedRoles,
251    sem: Arc<Semaphore>,
252    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
253    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
254    /// this is never the same handle as `interrupt`.
255    pause: Pause,
256    /// Set when `magi serve`'s interrupt scheduler wants this specific run
257    /// parked at its next node boundary, to let a task marked
258    /// [`crate::queue::Task::interrupt`] run alone before this one carries
259    /// on. Unlike `pause`, a fresh, unshared handle per run - see
260    /// [`Runner::watch_interrupt`].
261    interrupt: Pause,
262}
263
264/// The commit a run branches from: the base branch as the remote has it.
265///
266/// Two failures this replaces. A run used to branch off `HEAD` and so refused
267/// to start on a dirty tree, which made `magi serve` decline every task for as
268/// long as the operator had work in progress - most of the time. Branching off
269/// the *local* base branch fixed that and introduced a worse one: `land` merges
270/// the winner on GitHub, nothing updates the local ref, and the next run
271/// branches off a base missing everything the previous runs landed. Two tasks
272/// in a row from a phone would have had the second silently re-implementing
273/// against stale code and opening a pull request that reverted the first.
274///
275/// Only refs move here - no checkout, no local branch, no merge - so it is safe
276/// with uncommitted work in the tree. A machine with no network still starts:
277/// the fetch may fail and the local tip is used with a warning, because
278/// refusing to run offline is a worse failure than running against a base the
279/// operator can see for themselves.
280///
281/// One function, called by both entry points. Two answers to "where does a run
282/// branch from" is the kind of drift nobody notices until a diff is wrong.
283async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
284    let tracking = format!("{remote}/{base_branch}");
285    let fetched = git::fetch(repo, remote, base_branch).await;
286    if let Ok(out) = &fetched
287        && out.ok()
288        && git::rev_exists(repo, &tracking).await
289    {
290        return git::rev_parse(repo, &tracking).await;
291    }
292    let why = match &fetched {
293        Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
294        Ok(_) => format!("{remote} has no {base_branch}"),
295        Err(e) => e.to_string(),
296    };
297    tracing::warn!(
298        "could not read {tracking} ({why}); branching off the local \
299         {base_branch} instead, which may be behind"
300    );
301    git::rev_parse(repo, base_branch).await.with_context(|| {
302        format!(
303            "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
304             branch that exists"
305        )
306    })
307}
308
309impl Runner {
310    /// Start a fresh run against `repo`.
311    pub async fn start(repo: &Path, instruction: String, config: Config) -> Result<Self> {
312        let repo = git::toplevel(repo).await?;
313        let missing = agent::missing_programs(&config.agents);
314        if !missing.is_empty() {
315            bail!(
316                "these agent programs are not on PATH: {}. Fix the roster in \
317                 magi.toml or install them.",
318                missing.join(", ")
319            );
320        }
321        let base_branch = match config.merge.base.clone() {
322            Some(b) => b,
323            None => git::current_branch(&repo)
324                .await?
325                .context("HEAD is detached; set [merge] base in magi.toml")?,
326        };
327        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
328        // Still worth saying out loud. The operator's uncommitted work is not
329        // part of this run, and someone watching a candidate fail to use a
330        // change they just made deserves to know why.
331        if !git::is_clean(&repo).await? {
332            tracing::warn!(
333                "{} has uncommitted changes; they are not part of this run, \
334                 which branches off {base_branch} ({})",
335                repo.display(),
336                &base_commit[..base_commit.len().min(8)]
337            );
338        }
339        let roles = config.resolve_roles()?;
340        let max_parallel = config.graph.max_parallel.max(1);
341        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
342        state.event("start", format!("run {} created", state.id));
343        state.save()?;
344        Ok(Self {
345            state,
346            roles,
347            sem: Arc::new(Semaphore::new(max_parallel)),
348            pause: Pause::new(),
349            interrupt: Pause::new(),
350        })
351    }
352
353    /// Open a review-only run against work that already exists on `branch`.
354    ///
355    /// The expensive half of the graph is the implement wave — measured at
356    /// 111 and 134 internal tool-loop turns on this repository, against a
357    /// handful for a judge or a reviewer. The cheap half is worth running on
358    /// hand-written work too, and there was no way to reach it.
359    ///
360    /// No new state and no schema change are needed: a run with **one** viable
361    /// candidate and a tally already decided degrades `execute` to exactly
362    /// review → gate → merge, because `judge` skips a single-candidate field,
363    /// `deliberate` has fewer than two first choices to reconcile, `vote`
364    /// returns early, `tally` is already present and `fold_losers` has no
365    /// losers. Resuming such a run therefore does the right thing as well.
366    pub async fn review(repo: &Path, branch: &str, config: Config) -> Result<Self> {
367        let repo = git::toplevel(repo).await?;
368        let missing = agent::missing_programs(&config.agents);
369        if !missing.is_empty() {
370            bail!(
371                "these agent programs are not on PATH: {}. Fix the roster in \
372                 magi.toml or install them.",
373                missing.join(", ")
374            );
375        }
376        if !git::branch_exists(&repo, branch).await? {
377            bail!("no branch `{branch}` in {}", repo.display());
378        }
379        let base_branch = match config.merge.base.clone() {
380            Some(b) => b,
381            None => git::current_branch(&repo)
382                .await?
383                .context("HEAD is detached; set [merge] base in magi.toml")?,
384        };
385        if base_branch == branch {
386            bail!("`{branch}` is the base branch; there is nothing to review against");
387        }
388        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
389
390        let roles = config.resolve_roles()?;
391        let max_parallel = config.graph.max_parallel.max(1);
392        // The commit subjects are the closest thing to a task statement that
393        // existing work carries, and the reviewers are told as much.
394        let log = git::log_oneline(&repo, &base_commit, branch)
395            .await
396            .unwrap_or_default();
397        let instruction = format!(
398            "Review the work already on branch `{branch}`. There is no task \
399             statement: what the change claims to do is whatever its commits \
400             say.\n\n{}",
401            if log.trim().is_empty() {
402                "(no commit messages)"
403            } else {
404                log.trim()
405            }
406        );
407        let mut state = RunState::new(
408            repo.clone(),
409            base_branch,
410            base_commit.clone(),
411            instruction,
412            config,
413        );
414
415        // An attached worktree, so the fixer's commits land on the branch under
416        // review rather than on a detached head nobody will look at again.
417        let worktree = state.worktree_root().join("under-review");
418        if let Some(parent) = worktree.parent() {
419            tokio::fs::create_dir_all(parent).await.ok();
420        }
421        let path = worktree.to_string_lossy().to_string();
422        git::git(&repo, &["worktree", "add", &path, branch])
423            .await
424            .with_context(|| {
425                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
426            })?;
427
428        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
429            .await
430            .unwrap_or(0);
431        if commits == 0 {
432            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
433        }
434        let files = git::changed_files(&worktree, &base_commit, "HEAD")
435            .await
436            .map(|f| f.len())
437            .unwrap_or(0);
438        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
439            .await
440            .unwrap_or_default();
441
442        state.candidates.push(Candidate {
443            index: 0,
444            label: 'A',
445            // Not an agent id on purpose: nothing in the roster wrote this, and
446            // the stats tables must not credit anyone with a win for it.
447            agent: "(existing branch)".to_owned(),
448            branch: branch.to_owned(),
449            worktree,
450            summary: String::new(),
451            stat,
452            files,
453            commits,
454            empty: false,
455            failed: None,
456            duration_ms: 0,
457            folded: false,
458        });
459        state.tally = Some(Tally {
460            first_choice: BTreeMap::from([('A', 0)]),
461            borda: BTreeMap::new(),
462            winner: 'A',
463            rankings: 0,
464            unanimous_initial: false,
465            deliberated: false,
466            changed_votes: 0,
467            unanimous_final: false,
468            tie_break: None,
469            // No panel sat, so no quorum applies. Zero judges is the correct
470            // number for work that never competed, and must not be reported as
471            // a collapsed panel.
472            judges: 0,
473            present: 0,
474            quorum: 0,
475            met_quorum: true,
476            uncontested: Some("review-only run: nothing competed".to_owned()),
477        });
478        state.status = RunStatus::Reviewing;
479        state.event(
480            "start",
481            format!(
482                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
483                state.id
484            ),
485        );
486        state.save()?;
487        Ok(Self {
488            state,
489            roles,
490            sem: Arc::new(Semaphore::new(max_parallel)),
491            pause: Pause::new(),
492            interrupt: Pause::new(),
493        })
494    }
495
496    /// Reopen an existing run.
497    pub fn resume(id: &str) -> Result<Self> {
498        let state = RunState::load(id)?;
499        let roles = state.config.resolve_roles()?;
500        let max_parallel = state.config.graph.max_parallel.max(1);
501        Ok(Self {
502            state,
503            roles,
504            sem: Arc::new(Semaphore::new(max_parallel)),
505            pause: Pause::new(),
506            interrupt: Pause::new(),
507        })
508    }
509
510    /// Walk the graph to a terminal state, skipping nodes already recorded.
511    pub async fn execute(&mut self) -> Result<()> {
512        // Moving again, so it is no longer parked. Set before the walk rather
513        // than in `resume`, so every way of re-entering the graph clears it
514        // and a card cannot claim a run is waiting to be resumed while the
515        // agents are already working.
516        self.state.parked = false;
517        // Any seat this state still lists as answering belongs to whatever
518        // process last drove this run — this one included, if it crashed
519        // mid-wave. Cleared and flushed immediately, before anything else
520        // runs, so a resume can never show a seat as live when nothing is
521        // asking it anything yet; the node that actually dispatches the next
522        // wave repopulates it.
523        if self.state.clear_active() {
524            self.state.save()?;
525        }
526        // A run that already lost its quorum never resumes into the verdict
527        // machinery: `deliberate` and `vote` would otherwise clobber the
528        // stalled marker back to Voting and the run would keep going past a
529        // verdict that is no longer trustworthy. Everything already recorded is
530        // kept, so the run stays resumable (or foldable) for a human to pick up.
531        //
532        // On --resume the run gets one chance to repair itself: the seats a
533        // rate limit took out are re-asked. If their quota has since reset and
534        // the quorum is restored, the run picks up and finishes; otherwise it
535        // stays stale and still-resumable for a later retry. If it does not
536        // recover, the returned status stays `Stalled` and nothing was
537        // clobbered (the recovery only mutates entries for the lost seats).
538        if self.state.status == RunStatus::Stalled {
539            if self.recover_stall().await? {
540                self.finish_after_tally().await?;
541            } else {
542                // Still below quorum: persist the marker and stay resumable.
543                self.state.save()?;
544            }
545            return Ok(());
546        }
547        // A run parked inside `land` - watching CI, mid fix-round, or
548        // waiting on the owner's merge approval - resumes directly into it,
549        // never back through `prep`. Everything before `merge` already
550        // concluded; that is the only way `status` reaches `Landing` in the
551        // first place. Re-walking `review_loop` first would also be actively
552        // wrong: its own status recomputation (see its doc) treats any
553        // clean round as reason to set `status` to `Gating`, which would
554        // clobber this marker before `merge` ever ran, and this run would
555        // never find its way back into `land` at all.
556        if self.state.status == RunStatus::Landing {
557            self.run_land().await?;
558            // `run_land` may have settled the run right here - CI came back
559            // green and the PR merged, say - without ever passing back
560            // through `merge`'s own trailing call. Whatever it left `status`
561            // as is what this has to read.
562            self.settle_questions();
563            return Ok(());
564        }
565        self.prep().await?;
566        if self.park_here()? {
567            return Ok(());
568        }
569        self.advise().await?;
570        if self.park_here()? {
571            return Ok(());
572        }
573        self.implement().await?;
574        if self.park_here()? {
575            return Ok(());
576        }
577        self.judge().await?;
578        if self.park_here()? {
579            return Ok(());
580        }
581        self.deliberate().await?;
582        if self.park_here()? {
583            return Ok(());
584        }
585        self.vote().await?;
586        if self.park_here()? {
587            return Ok(());
588        }
589        self.tally()?;
590        // A verdict that lost its quorum is not trustworthy: do not review,
591        // gate, or merge on it. Everything already done is kept, so the run
592        // stays resumable (or foldable); the human can replace the agent that
593        // ran out of quota and pick it up.
594        if self.state.status == RunStatus::Stalled {
595            // Persist the stalled marker now — the normal end-of-execute save
596            // below is below this early return, and without it a resumed run
597            // would reload a pre-tally status and keep going.
598            self.state.save()?;
599            return Ok(());
600        }
601        self.finish_after_tally().await?;
602        Ok(())
603    }
604
605    /// Park here if asked to, recording it in the run's own timeline.
606    ///
607    /// Returns whether the caller should stop walking the graph. The state is
608    /// saved either way by the node that just finished; this adds the event so
609    /// the operator's card says why a run that is neither finished nor moving
610    /// is sitting where it is.
611    fn park_here(&mut self) -> Result<bool> {
612        // Either handle asking is enough - see `Pause`'s own doc for why
613        // they are never the same one. `interrupt` is checked second so a
614        // reason it carries is preferred in the message below over a plain
615        // shutdown park racing it at the same boundary.
616        if !self.pause.parked() && !self.interrupt.parked() {
617            return Ok(false);
618        }
619        let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
620            Some(reason) => format!(
621                "parked after `{}` ({reason}) — resume to carry on from here",
622                self.state.status.as_str()
623            ),
624            None => format!(
625                "parked after `{}` — resume to carry on from here",
626                self.state.status.as_str()
627            ),
628        };
629        self.state.event("park", why);
630        self.state.parked = true;
631        self.state.save()?;
632        Ok(true)
633    }
634
635    /// Hand the runner the pause `magi serve`'s own shutdown watches.
636    pub fn on_pause(&mut self, pause: Pause) {
637        self.pause = pause;
638    }
639
640    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
641    /// scheduler asking this one run - and no other - to park so a task
642    /// marked [`crate::queue::Task::interrupt`] can run alone. See
643    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
644    /// handle.
645    pub fn watch_interrupt(&mut self, pause: Pause) {
646        self.interrupt = pause;
647    }
648
649    /// Abandon this run's own open questions, once `status` has actually
650    /// settled rather than merely paused.
651    ///
652    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
653    /// either back up with the candidates, the review round and the seat
654    /// sessions already on disk, so a question an implementer asked mid-round
655    /// may still get a real answer read by a real resume. Only the three
656    /// statuses `resumable` excludes are actually final: the run merged, or
657    /// it reached `Ready` with nothing left to do, or it failed outright with
658    /// no established point to continue from. In every one of those the seat
659    /// that asked is gone for good, exactly like the run being deleted under
660    /// `magi run rm` - so the same cleanup applies, worded for what actually
661    /// happened instead of "the run was deleted".
662    ///
663    /// Best-effort and silent on success: called from every place `status`
664    /// can land on one of those three, including ones a resumed run revisits,
665    /// so it must cost nothing when there was nothing open to begin with.
666    fn settle_questions(&mut self) {
667        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
668            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
669        }
670    }
671
672    /// The tail of the graph after a trustworthy tally: fold losers, review,
673    /// gate, merge, and persist.
674    async fn finish_after_tally(&mut self) -> Result<()> {
675        self.fold_losers().await?;
676        // Before review starts, and again right before the gate: a run's
677        // review rounds can themselves take long enough for the base to move
678        // a second time, and the gate is the one node whose "green" gets
679        // acted on.
680        self.sync_to_base().await?;
681        self.review_loop().await?;
682        self.sync_to_base().await?;
683        self.gate().await?;
684        self.merge().await?;
685        self.state.save()?;
686        Ok(())
687    }
688
689    // ---------------------------------------------------------------- prep
690
691    async fn prep(&mut self) -> Result<()> {
692        if !self.state.candidates.is_empty() {
693            return Ok(());
694        }
695        self.state.status = RunStatus::Prep;
696        let repo = self.state.repo.clone();
697        let base = self.state.base_commit.clone();
698        let root = self.state.worktree_root();
699        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
700
701        // The hook is the write-time half of the blindness contract; the
702        // presentation filter in `blind` is the half that cannot be bypassed.
703        let hooks_dir = self.state.dir().join("hooks");
704        if self.state.config.blind.commit_msg_hook {
705            std::fs::create_dir_all(&hooks_dir)
706                .with_context(|| format!("create {}", hooks_dir.display()))?;
707            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
708            let path = hooks_dir.join("commit-msg");
709            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
710            make_executable(&path)?;
711            // Ref-counted rather than a plain idempotent set: with more than
712            // one run able to be in flight in the same repository at once
713            // (see `Config::daemon.max_concurrent_runs`), a bare "already
714            // true?" check cannot tell "another run of mine still needs
715            // this" from "nobody does", and the run that happens to finish
716            // first would disable the hook out from under a sibling still
717            // relying on it.
718            git::acquire_worktree_config(&repo).await?;
719            self.state.enabled_worktree_config = true;
720        }
721
722        for (index, (spec, label)) in self
723            .roles
724            .implementers
725            .clone()
726            .into_iter()
727            .zip(labels)
728            .enumerate()
729        {
730            let branch = self.state.branch_for(label);
731            let worktree = root.join(format!("cand-{label}"));
732            git::worktree_add_branch(&repo, &worktree, &branch, &base).await?;
733            if self.state.config.blind.commit_msg_hook {
734                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
735            }
736            git::local_exclude(&worktree, "/.magi/").await?;
737            self.state.candidates.push(Candidate {
738                index,
739                label,
740                agent: spec.id.clone(),
741                branch,
742                worktree,
743                summary: String::new(),
744                stat: String::new(),
745                files: 0,
746                commits: 0,
747                empty: false,
748                failed: None,
749                duration_ms: 0,
750                folded: false,
751            });
752        }
753
754        for j in 1..=self.roles.judges.len() {
755            let wt = root.join(format!("judge-{j}"));
756            if !wt.exists() {
757                git::worktree_add_detached(&repo, &wt, &base).await?;
758            }
759        }
760
761        // Disposable, detached checkouts for the design-deliberation stage's
762        // advisor seats — the same shape as the judges' above, at the same
763        // base commit, since advisors also only ever read. Sized off the
764        // configured count directly rather than a resolved roster: unlike
765        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
766        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
767        // `prep` has no `ResolvedRoles` field to read a count from here.
768        if self.state.config.graph.advise {
769            for k in 1..=self.state.config.graph.advisors {
770                let wt = root.join(format!("advisor-{k}"));
771                if !wt.exists() {
772                    git::worktree_add_detached(&repo, &wt, &base).await?;
773                }
774            }
775        }
776
777        // A judge cannot tell it is looking at its own patch — the seats keep
778        // separate conversations — but a panel that shares agents with the
779        // field is less independent than it looks, and that is worth saying out
780        // loud once per run rather than leaving it in the config.
781        let authors: Vec<&str> = self
782            .roles
783            .implementers
784            .iter()
785            .map(|a| a.id.as_str())
786            .collect();
787        let overlap: Vec<String> = self
788            .roles
789            .judges
790            .iter()
791            .enumerate()
792            .filter(|(_, j)| authors.contains(&j.id.as_str()))
793            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
794            .collect();
795        if !overlap.is_empty() {
796            let note = format!(
797                "{} also authored a candidate; blind, but the panel is less \
798                 independent than {} distinct agents would be",
799                overlap.join(", "),
800                self.roles.judges.len()
801            );
802            self.state.event("prep", note);
803        }
804
805        self.state.event(
806            "prep",
807            format!(
808                "{} candidates, {} judges, base {} ({})",
809                self.state.candidates.len(),
810                self.roles.judges.len(),
811                &self.state.base_commit[..7.min(self.state.base_commit.len())],
812                self.state.base_branch
813            ),
814        );
815        self.state.status = RunStatus::Implementing;
816        self.state.save()?;
817        Ok(())
818    }
819
820    // -------------------------------------------------------------- advise
821
822    /// The design-deliberation stage: independent, read-only advisor seats
823    /// each sketch a design before any implementer touches the repository,
824    /// and (when at least one produced a usable proposal) a synthesis seat
825    /// blends them into a brief `implement` carries in every candidate's
826    /// prompt.
827    ///
828    /// `[graph] advise` is the on/off switch, on by default; `[graph]
829    /// advisors` is the proposal count. Everything here is best-effort and
830    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
831    /// that cannot reach quota, or a synthesis seat that produced nothing
832    /// usable all leave `implement` exactly as it was before this stage
833    /// existed — the task instruction alone — rather than failing the whole
834    /// competition over an enrichment stage. Every outcome is still recorded
835    /// as an event, so a run that got nothing from this stage says why.
836    ///
837    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
838    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
839    /// resumed run whose stage failed would re-run it, and re-spend the
840    /// agent calls, on every reentry before `implement`.
841    ///
842    /// Also skipped once any candidate shows implementation progress — the
843    /// exact predicate `implement` itself uses to decide a candidate is no
844    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
845    /// is not enough: a run created by an older binary that predates this
846    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
847    /// an already-`Implementing`-or-later run under this build would
848    /// otherwise walk straight back through `prep` (a no-op once candidates
849    /// exist) into this node and spawn every advisor seat against worktrees
850    /// `prep` never recreated — after implementation has already started,
851    /// which is exactly the invariant this stage exists to guarantee.
852    async fn advise(&mut self) -> Result<()> {
853        let implement_untouched = self
854            .state
855            .candidates
856            .iter()
857            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
858        if !self.state.config.graph.advise || self.state.advise_attempted {
859            return Ok(());
860        }
861        if !implement_untouched {
862            self.state.event(
863                "advise",
864                "skipping the design-deliberation stage: at least one \
865                 candidate already shows implementation progress, so this \
866                 run is past the point the stage exists to run before"
867                    .to_owned(),
868            );
869            self.state.advise_attempted = true;
870            self.state.save()?;
871            return Ok(());
872        }
873        let run_id = self.state.id.clone();
874        let prompts = self.state.config.prompts.clone();
875        let instruction = self.state.instruction.clone();
876        let language = self.state.config.graph.language.clone();
877        let root = self.state.worktree_root();
878        let n = self.state.config.graph.advisors;
879        let where_recorded = self.state.dir().join("run.json");
880
881        let seats = match self.state.config.advisors() {
882            Ok(seats) if !seats.is_empty() => seats,
883            Ok(_) => {
884                self.state.event(
885                    "advise",
886                    format!(
887                        "[graph] advisors is 0; skipping the design-deliberation \
888                         stage and continuing without a synthesis brief (see {})",
889                        where_recorded.display()
890                    ),
891                );
892                self.state.advise_attempted = true;
893                self.state.save()?;
894                return Ok(());
895            }
896            Err(e) => {
897                self.state.event(
898                    "advise",
899                    format!(
900                        "could not resolve advisor seats ({e:#}); continuing \
901                         without a design-deliberation brief (see {})",
902                        where_recorded.display()
903                    ),
904                );
905                self.state.advise_attempted = true;
906                self.state.save()?;
907                return Ok(());
908            }
909        };
910
911        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
912        let artifacts = agent::artifacts_dir(&self.state.dir());
913        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
914
915        let mut jobs = Vec::new();
916        for (i, spec) in seats.iter().cloned().enumerate() {
917            let seat_key = format!("advisor-{}", i + 1);
918            let seat = self.seat(&seat_key, &spec.id);
919            jobs.push(SeatJob {
920                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
921                spec,
922                seat,
923                cwd: worktrees[i % worktrees.len()].clone(),
924                timeout,
925                allow_write: false,
926                sessions: false,
927                artifacts: artifacts.clone(),
928                stem: seat_key,
929            });
930        }
931
932        self.state.event(
933            "advise",
934            format!(
935                "{} advisor seat(s) sketching a design in parallel",
936                jobs.len()
937            ),
938        );
939        let mut quota_losses = Vec::new();
940        let cache = self.state.config.cache_dir();
941        let ctx = WaveCtx {
942            run: &run_id,
943            node: "advise",
944            prompts: &prompts,
945            cache: cache.as_deref(),
946        };
947        let results = ask_json_wave::<Proposal>(
948            jobs,
949            Arc::clone(&self.sem),
950            self.state.config.graph.retries,
951            &ctx,
952            &mut quota_losses,
953            &mut self.state,
954            &|p: &Proposal| p.validate(),
955        )
956        .await;
957        self.state.quota.extend(quota_losses);
958
959        let mut records = Vec::with_capacity(results.len());
960        for (i, (seat, res)) in results.into_iter().enumerate() {
961            let agent_id = seat.agent.clone();
962            self.state.seats.insert(seat.key.clone(), seat);
963            match res {
964                Ok((proposal, out)) => {
965                    self.state
966                        .event("advise", format!("advisor-{} proposed a design", i + 1));
967                    records.push(advise::AdvisorRecord::proposed(
968                        i + 1,
969                        agent_id,
970                        proposal,
971                        out.duration_ms,
972                    ));
973                }
974                Err(e) => {
975                    self.state.event(
976                        "advise",
977                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
978                    );
979                    records.push(advise::AdvisorRecord::failed(
980                        i + 1,
981                        agent_id,
982                        e.to_string(),
983                    ));
984                }
985            }
986        }
987
988        let mut advice = advise::Advice {
989            records,
990            synthesis: None,
991        };
992        if advice.proposals().is_empty() {
993            self.state.event(
994                "advise",
995                "no advisor produced a usable proposal; continuing without a \
996                 synthesis brief"
997                    .to_owned(),
998            );
999        } else {
1000            match self
1001                .synthesize_brief(
1002                    &advice,
1003                    &instruction,
1004                    &language,
1005                    &worktrees[0],
1006                    &artifacts,
1007                    &run_id,
1008                    &prompts,
1009                    cache.as_deref(),
1010                )
1011                .await
1012            {
1013                Ok(Some(text)) => {
1014                    self.state.event(
1015                        "advise",
1016                        "synthesized a design brief for the implementer".to_owned(),
1017                    );
1018                    advice.synthesis = Some(text);
1019                }
1020                Ok(None) => {
1021                    self.state.event(
1022                        "advise",
1023                        "the synthesis seat produced nothing usable; continuing \
1024                         without a design brief"
1025                            .to_owned(),
1026                    );
1027                }
1028                Err(e) => {
1029                    self.state.event(
1030                        "advise",
1031                        format!("could not synthesize a design brief: {e:#}"),
1032                    );
1033                }
1034            }
1035        }
1036        advise::apply_reflection(&mut advice);
1037
1038        self.state.advice = Some(advice);
1039        self.state.advise_attempted = true;
1040        self.state.save()?;
1041        Ok(())
1042    }
1043
1044    /// The synthesis seat: reads every advisor's proposal and blends them
1045    /// into the design brief `advise` stores on [`RunState::advice`]. Split
1046    /// out of [`Runner::advise`] only for readability — it is not called
1047    /// anywhere else.
1048    ///
1049    /// Picked the same way [`crate::talk`]'s standing conversation and
1050    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`] with no
1051    /// explicit id, rather than a dedicated `[roles]` entry — one more role
1052    /// to configure for a seat that runs once per run and, unlike the
1053    /// advisors it reads, never needs more than one.
1054    #[allow(clippy::too_many_arguments)]
1055    async fn synthesize_brief(
1056        &mut self,
1057        advice: &advise::Advice,
1058        instruction: &str,
1059        language: &str,
1060        cwd: &Path,
1061        artifacts: &Path,
1062        run_id: &str,
1063        prompts: &Prompts,
1064        cache: Option<&Path>,
1065    ) -> Result<Option<String>> {
1066        let spec = agent::pick(&self.state.config.agents, None, &agent::installed)?;
1067        let mut seat = self.seat("advise-synthesis", &spec.id);
1068        let proposals = advice.proposals();
1069        let mut prompt = prompt::with_overlay(
1070            prompt::synthesize_brief(instruction, &proposals, language),
1071            prompts.overlay("advise"),
1072        );
1073        if cache.is_some() {
1074            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
1075            // below — see `prompt::build_cache_note`'s doc for why telling a
1076            // read-only seat to build through the shared cache is exactly how
1077            // a sandbox's write refusal gets misread as a defect.
1078            prompt.push('\n');
1079            prompt.push_str(&prompt::build_cache_note("advise", false));
1080        }
1081        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1082        let out = agent::invoke(
1083            &spec,
1084            &mut seat,
1085            &Invocation {
1086                cwd,
1087                prompt: &prompt,
1088                timeout,
1089                allow_write: false,
1090                sessions: false,
1091                artifacts,
1092                stem: "advise-synthesis",
1093                run: run_id,
1094                node: "advise",
1095                cache_dir: None,
1096                attachments: &[],
1097            },
1098        )
1099        .await?;
1100        self.state.seats.insert(seat.key.clone(), seat);
1101        if !out.usable() {
1102            return Ok(None);
1103        }
1104        let text =
1105            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1106        Ok((!text.trim().is_empty()).then_some(text))
1107    }
1108
1109    // ----------------------------------------------------------- implement
1110
1111    async fn implement(&mut self) -> Result<()> {
1112        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1113        // agent files with `magi task add` name the run that paid for it. The
1114        // prompt overlay is cloned alongside it because the waves borrow it
1115        // while `self` is mutably borrowed by the node's own bookkeeping.
1116        let run_id = self.state.id.clone();
1117        let prompts = self.state.config.prompts.clone();
1118        let todo: Vec<usize> = self
1119            .state
1120            .candidates
1121            .iter()
1122            .enumerate()
1123            .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1124            .map(|(i, _)| i)
1125            .collect();
1126        if todo.is_empty() {
1127            return self.after_implement();
1128        }
1129        self.state.status = RunStatus::Implementing;
1130
1131        let language = self.state.config.graph.language.clone();
1132        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1133        let sessions = self.state.config.graph.sessions;
1134        let artifacts = agent::artifacts_dir(&self.state.dir());
1135        // The design-deliberation stage's blended brief, when `advise` found
1136        // one — carried into every implementer's prompt the same way
1137        // regardless of which candidate it is.
1138        let brief = self
1139            .state
1140            .advice
1141            .as_ref()
1142            .and_then(|a| a.synthesis.as_deref())
1143            .map(str::to_owned);
1144
1145        let mut jobs = Vec::new();
1146        for &i in &todo {
1147            let (index, label, worktree) = {
1148                let c = &self.state.candidates[i];
1149                (c.index, c.label, c.worktree.clone())
1150            };
1151            let spec = self.roles.implementers[index].clone();
1152            let seat_key = format!("impl-{label}");
1153            let seat = self.seat(&seat_key, &spec.id);
1154            let instruction = self.state.instruction.clone();
1155            jobs.push(SeatJob {
1156                spec,
1157                seat,
1158                prompt: prompt::implement(
1159                    &instruction,
1160                    &worktree.to_string_lossy(),
1161                    &language,
1162                    brief.as_deref(),
1163                ),
1164                cwd: worktree,
1165                timeout,
1166                allow_write: true,
1167                sessions,
1168                artifacts: artifacts.clone(),
1169                stem: format!("impl-{label}"),
1170            });
1171        }
1172
1173        self.state.event(
1174            "implement",
1175            format!("{} candidates in parallel", jobs.len()),
1176        );
1177        // Kept so a seat whose CLI hung up can be asked again from the same
1178        // job: `wave` consumes what it is given.
1179        let sent = jobs.clone();
1180        let cache = self.state.config.cache_dir();
1181        let ctx = WaveCtx {
1182            run: &run_id,
1183            node: "implement",
1184            prompts: &prompts,
1185            cache: cache.as_deref(),
1186        };
1187        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1188        self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1189            .await;
1190        self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1191            .await;
1192
1193        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1194            let seat_key = seat.key.clone();
1195            self.state.seats.insert(seat.key.clone(), seat);
1196            let label = self.state.candidates[i].label;
1197            let worktree = self.state.candidates[i].worktree.clone();
1198            let base = self.state.base_commit.clone();
1199
1200            let (summary, duration, failed) = match out {
1201                AgentOutcome::Ok(o) => {
1202                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1203                    let failed = (!o.usable()).then(|| {
1204                        if o.timed_out {
1205                            "agent timed out".to_owned()
1206                        } else {
1207                            format!("agent exited with {:?}", o.exit_code)
1208                        }
1209                    });
1210                    (text, o.duration_ms, failed)
1211                }
1212                // Left un-resumed by `resume_undelivered` (a dirty tree
1213                // already rescues the work, or there was no session left to
1214                // resume into) — reported like the ordinary failure it is,
1215                // never as if `o.text` (the CLI's raw error JSON) were an
1216                // answer.
1217                AgentOutcome::Dropped(o) => {
1218                    let why = o
1219                        .dropped
1220                        .as_ref()
1221                        .map(|d| d.why.as_str())
1222                        .unwrap_or("the CLI ended the stream without delivering its answer");
1223                    (
1224                        String::new(),
1225                        o.duration_ms,
1226                        Some(format!("the CLI dropped the stream ({why})")),
1227                    )
1228                }
1229                AgentOutcome::Quota(o) => {
1230                    self.state.quota.push(QuotaLoss {
1231                        seat: seat_key,
1232                        node: "implement".to_owned(),
1233                        at: Timestamp::now(),
1234                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1235                    });
1236                    (
1237                        String::new(),
1238                        o.duration_ms,
1239                        Some("rate limited (quota); produced no change".to_owned()),
1240                    )
1241                }
1242                AgentOutcome::Failed(e) => (String::new(), 0, Some(e)),
1243            };
1244
1245            // Rescue anything the agent edited but never committed: an
1246            // uncommitted candidate would silently be an empty one.
1247            let rescued = git::commit_all(
1248                &worktree,
1249                &format!("magi: candidate {label} (uncommitted work)"),
1250            )
1251            .await
1252            .unwrap_or(false);
1253            let commits = git::commits_ahead(&worktree, &base, "HEAD")
1254                .await
1255                .unwrap_or(0);
1256            let patch = git::diff(&worktree, &base, "HEAD")
1257                .await
1258                .unwrap_or_default();
1259            let stat = git::diff_stat(&worktree, &base, "HEAD")
1260                .await
1261                .unwrap_or_default();
1262            let files = git::changed_files(&worktree, &base, "HEAD")
1263                .await
1264                .map(|f| f.len())
1265                .unwrap_or(0);
1266            write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1267
1268            let c = &mut self.state.candidates[i];
1269            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1270            c.stat = stat;
1271            c.files = files;
1272            c.commits = commits;
1273            c.duration_ms = duration;
1274            c.empty = commits == 0 || patch.trim().is_empty();
1275            // An agent that failed but still produced a committed change stays
1276            // in the running: the patch is what gets judged, not the exit code.
1277            c.failed = match failed {
1278                Some(_) if c.empty => failed,
1279                _ => None,
1280            };
1281            let note = match (&c.failed, c.empty, rescued) {
1282                (Some(e), _, _) => format!("candidate {label}: {e}"),
1283                (None, true, _) => format!("candidate {label}: no change produced"),
1284                (None, false, true) => {
1285                    format!(
1286                        "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1287                    )
1288                }
1289                (None, false, false) => {
1290                    format!("candidate {label}: {files} files, {commits} commits")
1291                }
1292            };
1293            self.state.event("implement", note);
1294            self.state.save()?;
1295        }
1296
1297        self.after_implement()
1298    }
1299
1300    /// Ask again, once, for work a CLI did and then failed to hand over.
1301    ///
1302    /// [`agent::dropped_stream`] recognises the one shape observed: an error
1303    /// status with an empty response and a usage report showing output tokens,
1304    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
1305    /// seven minutes and 14,267 output tokens that arrived as an empty
1306    /// candidate, because `agy`'s own subscriber fell behind and hung up.
1307    ///
1308    /// Two conditions, and both matter:
1309    ///
1310    /// - **Only when the tree is untouched.** Often the agent has already
1311    ///   written its files and only the closing message was lost; the rescue
1312    ///   commit below picks that up and there is nothing to ask for. Re-asking
1313    ///   then would pay for a second implementation of work already on disk.
1314    /// - **Once.** A CLI that drops one stream can drop the next, and this
1315    ///   node is the most expensive in the graph.
1316    ///
1317    /// The re-ask is a resume, not a re-run: `has_context` is true because the
1318    /// dropped reply still carried its `conversation_id`, so the seat is asked
1319    /// to finish what it was doing rather than sent the whole task again. It
1320    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
1321    /// node's - for the same reason a re-ranked judge does: restating finished
1322    /// work is not the work.
1323    ///
1324    /// Unlike a quota this is worth retrying at all: a rate limit fails the
1325    /// same way until it resets, while an abandoned conversation is still
1326    /// there to be picked up.
1327    async fn resume_undelivered(
1328        &mut self,
1329        results: &mut [(usize, SeatState, AgentOutcome)],
1330        sent: &[SeatJob],
1331        prompts: &Prompts,
1332        run_id: &str,
1333    ) {
1334        for (wi, seat, out) in results.iter_mut() {
1335            let Some(dropped) = (match &*out {
1336                AgentOutcome::Dropped(o) => o.dropped.clone(),
1337                _ => None,
1338            }) else {
1339                continue;
1340            };
1341            let Some(job) = sent.get(*wi) else { continue };
1342            // Already on disk? Then only the closing message was lost.
1343            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1344                self.state.event(
1345                    "implement",
1346                    format!(
1347                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1348                         work is in the tree",
1349                        seat.key, dropped.output_tokens, dropped.why
1350                    ),
1351                );
1352                continue;
1353            }
1354            // The re-ask only makes sense as a resume: `resume_after_drop`
1355            // says nothing about the task, trusting the seat to still hold it.
1356            // Without a session to resume — sessions disabled, or this CLI's
1357            // drop shape happened not to carry a session id — that prompt
1358            // would open a brand-new conversation with no context at all,
1359            // which is worse than leaving this as the ordinary failure it
1360            // already is.
1361            if !has_context(&job.spec, seat, job.sessions) {
1362                self.state.event(
1363                    "implement",
1364                    format!(
1365                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1366                         is no session left to resume",
1367                        seat.key, dropped.output_tokens, dropped.why
1368                    ),
1369                );
1370                continue;
1371            }
1372            self.state.event(
1373                "implement",
1374                format!(
1375                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1376                     conversation",
1377                    seat.key, dropped.output_tokens, dropped.why
1378                ),
1379            );
1380            let mut retry = job.clone();
1381            retry.seat = seat.clone();
1382            retry.prompt = prompt::resume_after_drop(&dropped.why);
1383            retry.timeout = retry_budget(job.timeout, true);
1384            retry.stem = format!("{}-resume", job.stem);
1385            let cache = self.state.config.cache_dir();
1386            let ctx = WaveCtx {
1387                run: run_id,
1388                node: "implement",
1389                prompts,
1390                cache: cache.as_deref(),
1391            };
1392            let (resumed_seat, resumed) =
1393                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1394            *seat = resumed_seat;
1395            *out = resumed;
1396        }
1397    }
1398
1399    /// Ask an implement seat's own CLI to confirm what it started, once, when
1400    /// its reply reported a command whose completion status it never
1401    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
1402    /// that does and does not mean.
1403    ///
1404    /// The completion contract this task asks for, extended to `implement`
1405    /// with the same signal `continue_fix_report` reads for the fixer,
1406    /// rather than a keyword search over the reply or a hard requirement on
1407    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
1408    /// a candidate's CLI turn ended cleanly while a test run it had started
1409    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
1410    /// named in it at all is untouched by this: `commands` is empty, so
1411    /// there is nothing to be unconfirmed.
1412    ///
1413    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
1414    /// this is not about recovering edits that might already be on disk, it
1415    /// is about a result the seat itself never vouched for, which resuming
1416    /// asks for regardless of what the tree already holds. Bounded to one
1417    /// attempt for the same reason `resume_undelivered` is — this is the
1418    /// most expensive node in the graph — and a seat that still cannot
1419    /// confirm on that attempt is left as whatever its (possibly still
1420    /// unconfirmed) reply says; this does not invent a new "failed" reason
1421    /// for a candidate that otherwise produced a real, committed change.
1422    async fn resume_unconfirmed_commands(
1423        &mut self,
1424        results: &mut [(usize, SeatState, AgentOutcome)],
1425        sent: &[SeatJob],
1426        prompts: &Prompts,
1427        run_id: &str,
1428    ) {
1429        for (wi, seat, out) in results.iter_mut() {
1430            let AgentOutcome::Ok(o) = &*out else {
1431                continue;
1432            };
1433            if !has_unconfirmed_command(&o.commands) {
1434                continue;
1435            }
1436            let Some(job) = sent.get(*wi) else { continue };
1437            if !has_context(&job.spec, seat, job.sessions) {
1438                self.state.event(
1439                    "implement",
1440                    format!(
1441                        "{}: the reply named a command whose own CLI never confirmed the exit \
1442                         status of, but there is no session left to resume",
1443                        seat.key
1444                    ),
1445                );
1446                continue;
1447            }
1448            self.state.event(
1449                "implement",
1450                format!(
1451                    "{}: the reply named a command whose own CLI never confirmed the exit \
1452                     status of; resuming the conversation",
1453                    seat.key
1454                ),
1455            );
1456            let mut retry = job.clone();
1457            retry.seat = seat.clone();
1458            retry.prompt = prompt::resume_incomplete(
1459                "a command in your last reply had no confirmed exit status",
1460            );
1461            retry.timeout = retry_budget(job.timeout, true);
1462            retry.stem = format!("{}-confirm", job.stem);
1463            let cache = self.state.config.cache_dir();
1464            let ctx = WaveCtx {
1465                run: run_id,
1466                node: "implement",
1467                prompts,
1468                cache: cache.as_deref(),
1469            };
1470            let (resumed_seat, resumed) =
1471                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1472            *seat = resumed_seat;
1473            *out = resumed;
1474        }
1475    }
1476
1477    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
1478    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
1479    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
1480    /// that motivated this.
1481    ///
1482    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
1483    /// own nudge loop already covers for judge/review/vote seats, and not a
1484    /// dropped stream, which [`Runner::resume_undelivered`] covers for
1485    /// implement seats: here the CLI turn genuinely finished while the node's
1486    /// own work — the fixer's account of what it did — had not. Gated purely
1487    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
1488    /// reply, never on any wording in it, so a fixer whose valid, first-try
1489    /// `FixReport` happens to mention having waited on a background test is
1490    /// never resumed — the `Ok(report)` branch at the call site returns
1491    /// before this is ever invoked.
1492    ///
1493    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
1494    /// attempt ([`retry_budget`]), nothing attempted once the session is
1495    /// gone, and a quota hit ends the loop immediately rather than retrying a
1496    /// rate limit that fails the same way again.
1497    async fn continue_fix_report(
1498        &mut self,
1499        mut seat: SeatState,
1500        parse_err: String,
1501        job: &SeatJob,
1502        prompts: &Prompts,
1503        run_id: &str,
1504        round: usize,
1505    ) -> (
1506        SeatState,
1507        Option<FixReport>,
1508        Option<String>,
1509        ContinuationRecord,
1510    ) {
1511        let mut last_err = parse_err;
1512        let mut cumulative_wait_ms = 0u64;
1513        let mut attempts = 0usize;
1514        loop {
1515            if !has_context(&job.spec, &seat, job.sessions) {
1516                self.state.event(
1517                    "fix",
1518                    format!(
1519                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
1520                         session left to resume into"
1521                    ),
1522                );
1523                let outcome = if attempts == 0 {
1524                    ContinuationOutcome::NoSession
1525                } else {
1526                    ContinuationOutcome::Exhausted
1527                };
1528                return (
1529                    seat,
1530                    None,
1531                    Some(format!("unparsable fix report: {last_err}")),
1532                    ContinuationRecord {
1533                        attempts,
1534                        cumulative_wait_ms,
1535                        outcome,
1536                    },
1537                );
1538            }
1539            if attempts >= MAX_FIX_CONTINUATIONS {
1540                self.state.event(
1541                    "fix",
1542                    format!(
1543                        "round {round}: fixer's reply still had no adoption report after \
1544                         {attempts} continuation(s) ({last_err}); giving up"
1545                    ),
1546                );
1547                return (
1548                    seat,
1549                    None,
1550                    Some(format!(
1551                        "unparsable fix report after {attempts} continuation(s): {last_err}"
1552                    )),
1553                    ContinuationRecord {
1554                        attempts,
1555                        cumulative_wait_ms,
1556                        outcome: ContinuationOutcome::Exhausted,
1557                    },
1558                );
1559            }
1560            attempts += 1;
1561            self.state.event(
1562                "fix",
1563                format!(
1564                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
1565                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
1566                ),
1567            );
1568            let mut retry = job.clone();
1569            retry.seat = seat.clone();
1570            retry.prompt = prompt::resume_incomplete(&last_err);
1571            retry.timeout = retry_budget(job.timeout, true);
1572            retry.stem = format!("{}-continue{attempts}", job.stem);
1573            let cache = self.state.config.cache_dir();
1574            let ctx = WaveCtx {
1575                run: run_id,
1576                node: "fix",
1577                prompts,
1578                cache: cache.as_deref(),
1579            };
1580            let (resumed_seat, resumed_out) = run_one(
1581                retry,
1582                Arc::clone(&self.sem),
1583                &ctx,
1584                &mut self.state,
1585                attempts,
1586            )
1587            .await;
1588            seat = resumed_seat;
1589            match resumed_out {
1590                AgentOutcome::Ok(o) => {
1591                    cumulative_wait_ms += o.duration_ms;
1592                    match verdict::extract_json::<FixReport>(&o.text) {
1593                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
1594                            self.state.event(
1595                                "fix",
1596                                format!(
1597                                    "round {round}: fixer's adoption report recovered after \
1598                                     {attempts} continuation(s)"
1599                                ),
1600                            );
1601                            return (
1602                                seat,
1603                                Some(report),
1604                                None,
1605                                ContinuationRecord {
1606                                    attempts,
1607                                    cumulative_wait_ms,
1608                                    outcome: ContinuationOutcome::Resumed,
1609                                },
1610                            );
1611                        }
1612                        // The report parsed, but this same reply's own
1613                        // CommandEvidence — the identical record `state.jobs`
1614                        // renders — names a command whose CLI never
1615                        // confirmed an exit status. Read together, that is
1616                        // not a resolved answer: keep nudging rather than
1617                        // accept a report standing next to a command the
1618                        // seat's own CLI cannot vouch for.
1619                        Ok(_) => {
1620                            last_err = "the reply parsed, but it reported a command whose own CLI \
1621                                 never confirmed an exit status"
1622                                .to_owned();
1623                        }
1624                        Err(e) => last_err = e.to_string(),
1625                    }
1626                }
1627                AgentOutcome::Quota(o) => {
1628                    cumulative_wait_ms += o.duration_ms;
1629                    self.state.quota.push(QuotaLoss {
1630                        seat: seat.key.clone(),
1631                        node: "fix".to_owned(),
1632                        at: Timestamp::now(),
1633                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1634                    });
1635                    self.state.event(
1636                        "fix",
1637                        format!(
1638                            "round {round}: continuation rate limited (quota); not retrying now"
1639                        ),
1640                    );
1641                    return (
1642                        seat,
1643                        None,
1644                        Some("rate limited (quota) while recovering the fix report".to_owned()),
1645                        ContinuationRecord {
1646                            attempts,
1647                            cumulative_wait_ms,
1648                            outcome: ContinuationOutcome::QuotaLost,
1649                        },
1650                    );
1651                }
1652                AgentOutcome::Dropped(o) => {
1653                    cumulative_wait_ms += o.duration_ms;
1654                    let why = o
1655                        .dropped
1656                        .as_ref()
1657                        .map(|d| d.why.as_str())
1658                        .unwrap_or("the CLI ended the stream without delivering its answer");
1659                    last_err = format!("the CLI dropped the stream ({why})");
1660                }
1661                AgentOutcome::Failed(e) => last_err = e,
1662            }
1663        }
1664    }
1665
1666    fn after_implement(&mut self) -> Result<()> {
1667        // Scan every candidate patch once the set is complete.
1668        if self.state.leaks.is_empty() {
1669            let cfg = self.state.config.blind.clone();
1670            let mut leaks = Vec::new();
1671            for c in &self.state.candidates {
1672                let Some(patch) =
1673                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
1674                else {
1675                    continue;
1676                };
1677                leaks.extend(blind::scan(
1678                    &format!("candidate {} patch", c.label),
1679                    &patch,
1680                    &cfg.vendor_tokens,
1681                ));
1682            }
1683            if !leaks.is_empty() {
1684                let summary = leaks
1685                    .iter()
1686                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
1687                    .collect::<Vec<_>>()
1688                    .join(", ");
1689                match cfg.on_leak {
1690                    LeakPolicy::Fail => {
1691                        self.state.status = RunStatus::Failed;
1692                        self.state
1693                            .event("blind", format!("vendor text in a patch: {summary}"));
1694                        self.state.leaks = leaks;
1695                        self.state.save()?;
1696                        self.settle_questions();
1697                        bail!(
1698                            "blind.on_leak = \"fail\" and vendor text reached a \
1699                             judged patch: {summary}"
1700                        );
1701                    }
1702                    LeakPolicy::Redact => self.state.event(
1703                        "blind",
1704                        format!("redacting vendor text for judging: {summary}"),
1705                    ),
1706                    LeakPolicy::Warn => self.state.event(
1707                        "blind",
1708                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
1709                    ),
1710                }
1711                self.state.leaks = leaks;
1712            }
1713        }
1714
1715        if self.state.viable().is_empty() {
1716            self.state.status = RunStatus::Failed;
1717            self.state.save()?;
1718            self.settle_questions();
1719            bail!("no candidate produced a change; nothing to judge");
1720        }
1721        self.state.status = RunStatus::Judging;
1722        self.state.save()?;
1723        Ok(())
1724    }
1725
1726    // --------------------------------------------------------------- judge
1727
1728    async fn judge(&mut self) -> Result<()> {
1729        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1730        // agent files with `magi task add` name the run that paid for it. The
1731        // prompt overlay is cloned alongside it because the waves borrow it
1732        // while `self` is mutably borrowed by the node's own bookkeeping.
1733        let run_id = self.state.id.clone();
1734        let prompts = self.state.config.prompts.clone();
1735        if !self.state.judgements.is_empty() || self.state.judge_skipped {
1736            return Ok(());
1737        }
1738        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1739        if viable.len() == 1 {
1740            // Recorded so this is a one-time event: `judgements` stays empty
1741            // either way, which without this flag is indistinguishable from
1742            // "not yet judged" on the next reentry — and status is left
1743            // untouched, so a later node's conclusion (e.g. `Blocked` after
1744            // the review budget ran out) survives a resume instead of being
1745            // clobbered back to `Judging` by this node running again.
1746            self.state.judge_skipped = true;
1747            self.state.event(
1748                "judge",
1749                format!(
1750                    "only candidate {} produced a change; judging skipped",
1751                    viable[0].label
1752                ),
1753            );
1754            self.state.save()?;
1755            return Ok(());
1756        }
1757        self.state.status = RunStatus::Judging;
1758
1759        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
1760        let language = self.state.config.graph.language.clone();
1761        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1762        let sessions = self.state.config.graph.sessions;
1763        let artifacts = agent::artifacts_dir(&self.state.dir());
1764        let root = self.state.worktree_root();
1765        let base_short = short(&self.state.base_commit);
1766
1767        let mut jobs = Vec::new();
1768        let mut orders = Vec::new();
1769        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1770            let order = blind::presentation_order(viable.len(), j, self.state.seed);
1771            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
1772            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
1773            let seat_key = format!("judge-{}", j + 1);
1774            let seat = self.seat(&seat_key, &spec.id);
1775            jobs.push(SeatJob {
1776                prompt: prompt::judge(
1777                    &self.state.instruction,
1778                    &views,
1779                    self.roles.judges.len(),
1780                    &base_short,
1781                    &language,
1782                ),
1783                spec,
1784                seat,
1785                cwd: root.join(format!("judge-{}", j + 1)),
1786                timeout,
1787                allow_write: false,
1788                sessions,
1789                artifacts: artifacts.clone(),
1790                stem: format!("judge-{}", j + 1),
1791            });
1792        }
1793
1794        self.state.event(
1795            "judge",
1796            format!(
1797                "{} judges ranking {} candidates blind",
1798                jobs.len(),
1799                viable.len()
1800            ),
1801        );
1802        let labels_for_check = labels.clone();
1803        let mut quota_losses = Vec::new();
1804        let cache = self.state.config.cache_dir();
1805        let ctx = WaveCtx {
1806            run: &run_id,
1807            node: "judge",
1808            prompts: &prompts,
1809            cache: cache.as_deref(),
1810        };
1811        let results = ask_json_wave::<Ranking>(
1812            jobs,
1813            Arc::clone(&self.sem),
1814            self.state.config.graph.retries,
1815            &ctx,
1816            &mut quota_losses,
1817            &mut self.state,
1818            &move |r: &Ranking| r.validate(&labels_for_check),
1819        )
1820        .await;
1821        self.state.quota.extend(quota_losses);
1822
1823        for (j, (seat, res)) in results.into_iter().enumerate() {
1824            let agent_id = seat.agent.clone();
1825            self.state.seats.insert(seat.key.clone(), seat);
1826            let mut record = Judgement {
1827                judge: j + 1,
1828                seat: format!("judge-{}", j + 1),
1829                agent: agent_id,
1830                ranking: Vec::new(),
1831                reasons: BTreeMap::new(),
1832                confidence: None,
1833                order: orders[j].clone(),
1834                failed: None,
1835                duration_ms: 0,
1836            };
1837            match res {
1838                Ok((ranking, out)) => {
1839                    record.ranking = ranking.normalized();
1840                    record.reasons = ranking.reasons;
1841                    record.confidence = ranking.confidence;
1842                    record.duration_ms = out.duration_ms;
1843                    self.state.event(
1844                        "judge",
1845                        format!(
1846                            "judge {} ranked {}",
1847                            j + 1,
1848                            record.ranking.iter().collect::<String>()
1849                        ),
1850                    );
1851                }
1852                Err(e) => {
1853                    record.failed = Some(e.to_string());
1854                    self.state
1855                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
1856                }
1857            }
1858            self.state.judgements.push(record);
1859            self.state.save()?;
1860        }
1861        Ok(())
1862    }
1863
1864    // ---------------------------------------------------------- deliberate
1865
1866    async fn deliberate(&mut self) -> Result<()> {
1867        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1868        // agent files with `magi task add` name the run that paid for it. The
1869        // prompt overlay is cloned alongside it because the waves borrow it
1870        // while `self` is mutably borrowed by the node's own bookkeeping.
1871        let run_id = self.state.id.clone();
1872        let prompts = self.state.config.prompts.clone();
1873        if !self.state.deliberation.is_empty() {
1874            return Ok(());
1875        }
1876        let tops: Vec<char> = self
1877            .state
1878            .judgements
1879            .iter()
1880            .filter_map(|j| j.ranking.first().copied())
1881            .collect();
1882        let rounds = self.state.config.graph.deliberate_rounds;
1883        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
1884            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
1885                self.state.event(
1886                    "deliberate",
1887                    format!("judges agreed on {} outright; no deliberation", tops[0]),
1888                );
1889            }
1890            self.state.status = RunStatus::Voting;
1891            self.state.save()?;
1892            return Ok(());
1893        }
1894
1895        self.state.status = RunStatus::Deliberating;
1896        self.state.event(
1897            "deliberate",
1898            format!(
1899                "split: first choices were {} — opening {rounds} round(s)",
1900                tops.iter().collect::<String>()
1901            ),
1902        );
1903
1904        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1905        let language = self.state.config.graph.language.clone();
1906        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1907        let sessions = self.state.config.graph.sessions;
1908        let artifacts = agent::artifacts_dir(&self.state.dir());
1909        let root = self.state.worktree_root();
1910        let base_short = short(&self.state.base_commit);
1911
1912        // Judges argue in sequence so that a turn can answer the one before it;
1913        // that is the difference between deliberation and three parallel
1914        // monologues.
1915        for round in 1..=rounds {
1916            let mut turns: Vec<DeliberationTurn> = Vec::new();
1917            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1918                if self.state.judgements[j].failed.is_some() {
1919                    continue;
1920                }
1921                let seat_key = format!("judge-{}", j + 1);
1922                let mut seat = self.seat(&seat_key, &spec.id);
1923                let transcript = self.transcript(&turns, j);
1924                let context = if has_context(&spec, &seat, sessions) {
1925                    None
1926                } else {
1927                    Some(self.candidate_block(&viable, &base_short))
1928                };
1929                let text = prompt::deliberate(
1930                    &self.state.instruction,
1931                    context.as_deref(),
1932                    &transcript,
1933                    round,
1934                    rounds,
1935                    &language,
1936                );
1937                let job = SeatJob {
1938                    spec,
1939                    seat: seat.clone(),
1940                    prompt: text,
1941                    cwd: root.join(format!("judge-{}", j + 1)),
1942                    timeout,
1943                    allow_write: false,
1944                    sessions,
1945                    artifacts: artifacts.clone(),
1946                    stem: format!("delib-{round}-judge-{}", j + 1),
1947                };
1948                let cache = self.state.config.cache_dir();
1949                let ctx = WaveCtx {
1950                    run: &run_id,
1951                    node: "deliberate",
1952                    prompts: &prompts,
1953                    cache: cache.as_deref(),
1954                };
1955                let (updated, out) =
1956                    run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1957                seat = updated;
1958                let agent_id = seat.agent.clone();
1959                let seat_key = seat.key.clone();
1960                self.state.seats.insert(seat.key.clone(), seat);
1961                let body = match out {
1962                    AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
1963                    // Never read the CLI's raw error JSON as this judge's
1964                    // position — skip the seat instead, the same as any other
1965                    // failed turn.
1966                    AgentOutcome::Dropped(o) => {
1967                        let why =
1968                            o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
1969                                "the CLI ended the stream without delivering its answer",
1970                            );
1971                        self.state.event(
1972                            "deliberate",
1973                            format!(
1974                                "judge {} skipped: the CLI dropped the stream ({why})",
1975                                j + 1
1976                            ),
1977                        );
1978                        continue;
1979                    }
1980                    AgentOutcome::Quota(o) => {
1981                        self.state.quota.push(QuotaLoss {
1982                            seat: seat_key,
1983                            node: "deliberate".to_owned(),
1984                            at: Timestamp::now(),
1985                            reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1986                        });
1987                        self.state.event(
1988                            "deliberate",
1989                            format!("judge {} skipped: rate limited (quota)", j + 1),
1990                        );
1991                        continue;
1992                    }
1993                    AgentOutcome::Failed(e) => {
1994                        self.state
1995                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
1996                        continue;
1997                    }
1998                };
1999                let tentative = verdict::extract_json::<Position>(&body)
2000                    .ok()
2001                    .and_then(|p| p.tentative)
2002                    .and_then(|s| s.trim().chars().next())
2003                    .map(|c| c.to_ascii_uppercase());
2004                self.state.event(
2005                    "deliberate",
2006                    format!(
2007                        "round {round}: judge {} now favours {}",
2008                        j + 1,
2009                        tentative.map_or("—".to_owned(), |c| c.to_string())
2010                    ),
2011                );
2012                turns.push(DeliberationTurn {
2013                    judge: j + 1,
2014                    agent: agent_id,
2015                    body: blind::sanitize_prose(&body, &self.state.config.blind),
2016                    tentative,
2017                });
2018            }
2019            self.state
2020                .deliberation
2021                .push(DeliberationRound { round, turns });
2022            self.state.save()?;
2023        }
2024
2025        self.state.status = RunStatus::Voting;
2026        self.state.save()?;
2027        Ok(())
2028    }
2029
2030    // ---------------------------------------------------------------- vote
2031
2032    async fn vote(&mut self) -> Result<()> {
2033        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2034        // agent files with `magi task add` name the run that paid for it. The
2035        // prompt overlay is cloned alongside it because the waves borrow it
2036        // while `self` is mutably borrowed by the node's own bookkeeping.
2037        let run_id = self.state.id.clone();
2038        let prompts = self.state.config.prompts.clone();
2039        if !self.state.votes.is_empty() {
2040            return Ok(());
2041        }
2042        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2043        if viable.len() == 1 {
2044            return Ok(());
2045        }
2046        self.state.status = RunStatus::Voting;
2047
2048        let language = self.state.config.graph.language.clone();
2049        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2050        let sessions = self.state.config.graph.sessions;
2051        let artifacts = agent::artifacts_dir(&self.state.dir());
2052        let root = self.state.worktree_root();
2053        let base_short = short(&self.state.base_commit);
2054        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2055
2056        let mut jobs = Vec::new();
2057        let mut seats_at = Vec::new();
2058        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2059            if self
2060                .state
2061                .judgements
2062                .get(j)
2063                .is_some_and(|r| r.failed.is_some())
2064            {
2065                continue;
2066            }
2067            let seat_key = format!("judge-{}", j + 1);
2068            let seat = self.seat(&seat_key, &spec.id);
2069            let mut text = prompt::final_vote(&viable, &language);
2070            if !has_context(&spec, &seat, sessions) {
2071                text = format!(
2072                    "{}\n\n# Candidates\n\n{}",
2073                    text,
2074                    self.candidate_block(&candidates, &base_short)
2075                );
2076            }
2077            jobs.push(SeatJob {
2078                spec,
2079                seat,
2080                prompt: text,
2081                cwd: root.join(format!("judge-{}", j + 1)),
2082                timeout,
2083                allow_write: false,
2084                sessions,
2085                artifacts: artifacts.clone(),
2086                stem: format!("vote-judge-{}", j + 1),
2087            });
2088            seats_at.push(j);
2089        }
2090
2091        self.state.event(
2092            "vote",
2093            format!(
2094                "collecting {} final votes one by one, privately",
2095                jobs.len()
2096            ),
2097        );
2098        let allowed = viable.clone();
2099        let mut quota_losses = Vec::new();
2100        let cache = self.state.config.cache_dir();
2101        let ctx = WaveCtx {
2102            run: &run_id,
2103            node: "vote",
2104            prompts: &prompts,
2105            cache: cache.as_deref(),
2106        };
2107        let results = ask_json_wave::<FinalVote>(
2108            jobs,
2109            Arc::clone(&self.sem),
2110            self.state.config.graph.retries,
2111            &ctx,
2112            &mut quota_losses,
2113            &mut self.state,
2114            &move |v: &FinalVote| match v.label() {
2115                Some(c) if allowed.contains(&c) => Ok(()),
2116                other => bail!("vote {other:?} is not one of {allowed:?}"),
2117            },
2118        )
2119        .await;
2120        self.state.quota.extend(quota_losses);
2121
2122        for (&j, (seat, res)) in seats_at.iter().zip(results) {
2123            let agent_id = seat.agent.clone();
2124            self.state.seats.insert(seat.key.clone(), seat);
2125            let initial = self
2126                .state
2127                .judgements
2128                .get(j)
2129                .and_then(|r| r.ranking.first().copied());
2130            let mut record = VoteRecord {
2131                judge: j + 1,
2132                agent: agent_id,
2133                vote: None,
2134                reason: String::new(),
2135                changed: false,
2136            };
2137            match res {
2138                Ok((v, _)) => {
2139                    record.vote = v.label();
2140                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2141                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
2142                    self.state.event(
2143                        "vote",
2144                        format!(
2145                            "judge {} voted {}{}",
2146                            j + 1,
2147                            record.vote.unwrap_or('?'),
2148                            if record.changed { " (changed)" } else { "" }
2149                        ),
2150                    );
2151                }
2152                Err(e) => {
2153                    self.state
2154                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
2155                }
2156            }
2157            self.state.votes.push(record);
2158            self.state.save()?;
2159        }
2160        Ok(())
2161    }
2162
2163    // --------------------------------------------------------------- tally
2164
2165    fn tally(&mut self) -> Result<()> {
2166        if self.state.tally.is_some() {
2167            return Ok(());
2168        }
2169        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2170        let tops: Vec<char> = self
2171            .state
2172            .judgements
2173            .iter()
2174            .filter_map(|j| j.ranking.first().copied())
2175            .collect();
2176        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
2177
2178        // A judge whose private vote failed still counted once, in the initial
2179        // ranking; using it beats discarding a whole seat.
2180        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2181        let mut cast: Vec<char> = Vec::new();
2182        for (i, j) in self.state.judgements.iter().enumerate() {
2183            let vote = self
2184                .state
2185                .votes
2186                .iter()
2187                .find(|v| v.judge == i + 1)
2188                .and_then(|v| v.vote)
2189                .or_else(|| j.ranking.first().copied());
2190            if let Some(v) = vote {
2191                *first_choice.entry(v).or_insert(0) += 1;
2192                cast.push(v);
2193            }
2194        }
2195
2196        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2197        for j in &self.state.judgements {
2198            let n = j.ranking.len();
2199            for (pos, label) in j.ranking.iter().enumerate() {
2200                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
2201            }
2202        }
2203
2204        let best = first_choice.values().copied().max().unwrap_or(0);
2205        let mut leaders: Vec<char> = first_choice
2206            .iter()
2207            .filter(|(_, v)| **v == best)
2208            .map(|(k, _)| *k)
2209            .collect();
2210        let mut tie_break = None;
2211        if leaders.len() > 1 {
2212            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
2213            let borda_leaders: Vec<char> = leaders
2214                .iter()
2215                .copied()
2216                .filter(|l| borda[l] == top_borda)
2217                .collect();
2218            tie_break = Some(if borda_leaders.len() == 1 {
2219                format!(
2220                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
2221                    leaders.len()
2222                )
2223            } else {
2224                format!(
2225                    "{} way tie on both first-choice votes and Borda points, broken by label order",
2226                    leaders.len()
2227                )
2228            });
2229            leaders = borda_leaders;
2230            leaders.sort_unstable();
2231        }
2232        let winner = *leaders
2233            .first()
2234            .or(viable.first())
2235            .context("no candidate to declare a winner from")?;
2236
2237        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
2238        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
2239        let deliberated = !self.state.deliberation.is_empty();
2240
2241        // Whose verdict is this? A rate-limited seat is absent even if it
2242        // ranked before the limit hit, so presence is measured against the
2243        // recorded losses, not just "did a ranking ever appear".
2244        let quota_seats: std::collections::BTreeSet<&str> =
2245            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2246        let mut present = 0usize;
2247        for (i, j) in self.state.judgements.iter().enumerate() {
2248            if quota_seats.contains(j.seat.as_str()) {
2249                continue;
2250            }
2251            let ranked = !j.ranking.is_empty() && j.failed.is_none();
2252            let voted = self
2253                .state
2254                .votes
2255                .iter()
2256                .any(|v| v.judge == i + 1 && v.vote.is_some());
2257            if ranked || voted {
2258                present += 1;
2259            }
2260        }
2261        // Strict majority of the configured panel. A bare majority is real
2262        // signal we can act on, while a minority verdict must never stand in
2263        // for a healthy one. A one-candidate run needs no panel at all, and
2264        // `judges` stays `0` rather than the roster size a panel that never
2265        // sat would otherwise be credited with.
2266        let needs_quorum = viable.len() > 1;
2267        let judges_total = if needs_quorum {
2268            self.roles.judges.len()
2269        } else {
2270            0
2271        };
2272        let quorum = if needs_quorum {
2273            judges_total / 2 + 1
2274        } else {
2275            0
2276        };
2277        let met_quorum = !needs_quorum || present >= quorum;
2278        let uncontested = (!needs_quorum).then(|| {
2279            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
2280        });
2281
2282        self.state.event(
2283            "tally",
2284            match &uncontested {
2285                Some(reason) => format!("winner {winner} — {reason}"),
2286                None => format!(
2287                    "winner {winner} — votes {} | initial {} | {} changed | \
2288                     {present}/{judges_total} judges{}",
2289                    first_choice
2290                        .iter()
2291                        .map(|(k, v)| format!("{k}:{v}"))
2292                        .collect::<Vec<_>>()
2293                        .join(" "),
2294                    if unanimous_initial {
2295                        "unanimous"
2296                    } else {
2297                        "split"
2298                    },
2299                    changed_votes,
2300                    if met_quorum {
2301                        String::new()
2302                    } else {
2303                        format!(" — below quorum ({quorum} required)")
2304                    },
2305                ),
2306            },
2307        );
2308        if !met_quorum {
2309            self.state.event(
2310                "stall",
2311                format!(
2312                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
2313                     the run stops here, resumable"
2314                ),
2315            );
2316        }
2317        self.state.tally = Some(Tally {
2318            first_choice,
2319            borda,
2320            winner,
2321            rankings: tops.len(),
2322            unanimous_initial,
2323            deliberated,
2324            changed_votes,
2325            unanimous_final,
2326            tie_break,
2327            judges: judges_total,
2328            present,
2329            quorum,
2330            met_quorum,
2331            uncontested,
2332        });
2333        self.state.status = if met_quorum {
2334            RunStatus::Reviewing
2335        } else {
2336            RunStatus::Stalled
2337        };
2338        self.state.save()?;
2339        Ok(())
2340    }
2341
2342    // ------------------------------------------------------------- recover
2343
2344    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
2345    /// resumed toward completion once the transient cause clears.
2346    ///
2347    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
2348    /// it toward the quorum, which is exactly the set of seats whose absence
2349    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
2350    /// not depend on which node happened to hit the limit), or an ordinary
2351    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
2352    /// seat is never disturbed.
2353    ///
2354    /// A seat that now answers with a usable ranking is "recovered": its
2355    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
2356    /// `tally` counts it present again), and its vote re-collected. A seat that
2357    /// still fails keeps its loss and stays absent.
2358    ///
2359    /// Returns `true` when the re-tally restores the quorum (the run may proceed
2360    /// to review/gate/merge), `false` when it is still below quorum (the run
2361    /// stays `Stalled`, still resumable for a later retry).
2362    #[allow(clippy::too_many_lines)]
2363    async fn recover_stall(&mut self) -> Result<bool> {
2364        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2365        // agent files with `magi task add` name the run that paid for it. The
2366        // prompt overlay is cloned alongside it because the waves borrow it
2367        // while `self` is mutably borrowed by the node's own bookkeeping.
2368        let run_id = self.state.id.clone();
2369        let prompts = self.state.config.prompts.clone();
2370        // Absent seats = quota-lost at any node, or failed outright. Mirroring
2371        // `tally`'s presence test (rather than the old quota-judge/vote filter)
2372        // is what keeps a non-quota collapse — or a quota loss recorded at the
2373        // deliberate node — from being a permanent dead-end on `--resume`.
2374        let quota_seats: BTreeSet<&str> =
2375            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2376        let absent: Vec<String> = self
2377            .state
2378            .judgements
2379            .iter()
2380            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
2381            .map(|j| j.seat.clone())
2382            .collect();
2383        if absent.is_empty() {
2384            return Ok(false);
2385        }
2386        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2387        if viable.len() <= 1 {
2388            return Ok(false);
2389        }
2390        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2391        let language = self.state.config.graph.language.clone();
2392        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2393        let sessions = self.state.config.graph.sessions;
2394        let artifacts = agent::artifacts_dir(&self.state.dir());
2395        let root = self.state.worktree_root();
2396        let base_short = short(&self.state.base_commit);
2397        let candidates: Vec<Candidate> = viable.clone();
2398
2399        // Map each absent seat key to its 0-based position in `roles.judges`.
2400        let mut positions: Vec<usize> = absent
2401            .iter()
2402            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
2403            .collect();
2404        if positions.is_empty() {
2405            return Ok(false);
2406        }
2407        positions.sort_unstable();
2408        positions.dedup();
2409
2410        // Re-rank the lost seats, one blind prompt each.
2411        let mut judge_jobs = Vec::new();
2412        for &j in &positions {
2413            let order = blind::presentation_order(viable.len(), j, self.state.seed);
2414            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2415            let seat_key = format!("judge-{}", j + 1);
2416            let spec = self.roles.judges[j].clone();
2417            let seat = self.seat(&seat_key, &spec.id);
2418            judge_jobs.push(SeatJob {
2419                spec,
2420                seat,
2421                prompt: prompt::judge(
2422                    &self.state.instruction,
2423                    &views,
2424                    self.roles.judges.len(),
2425                    &base_short,
2426                    &language,
2427                ),
2428                cwd: root.join(seat_key),
2429                timeout,
2430                allow_write: false,
2431                sessions,
2432                artifacts: artifacts.clone(),
2433                stem: format!("judge-{}-recover", j + 1),
2434            });
2435        }
2436
2437        let labels_for_check = labels.clone();
2438        let mut judge_losses = Vec::new();
2439        let retries = self.state.config.graph.retries;
2440        let cache = self.state.config.cache_dir();
2441        let ctx = WaveCtx {
2442            run: &run_id,
2443            node: "judge",
2444            prompts: &prompts,
2445            cache: cache.as_deref(),
2446        };
2447        let results = ask_json_wave::<Ranking>(
2448            judge_jobs,
2449            Arc::clone(&self.sem),
2450            retries,
2451            &ctx,
2452            &mut judge_losses,
2453            &mut self.state,
2454            &move |r: &Ranking| r.validate(&labels_for_check),
2455        )
2456        .await;
2457
2458        // Refresh the judgement of every seat that ranked again.
2459        let mut recovered: BTreeSet<usize> = BTreeSet::new();
2460        for (&j, (seat, res)) in positions.iter().zip(results) {
2461            self.state.seats.insert(seat.key.clone(), seat);
2462            let record = &mut self.state.judgements[j];
2463            match res {
2464                Ok((ranking, out)) => {
2465                    record.ranking = ranking.normalized();
2466                    record.reasons = ranking.reasons;
2467                    record.confidence = ranking.confidence;
2468                    record.failed = None;
2469                    record.duration_ms = out.duration_ms;
2470                    recovered.insert(j);
2471                    self.state.event(
2472                        "recover",
2473                        format!("judge {} ranked again after the limit", j + 1),
2474                    );
2475                }
2476                Err(e) => {
2477                    self.state
2478                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
2479                }
2480            }
2481        }
2482
2483        // Re-ask the votes of the seats that recovered a ranking.
2484        let mut vote_jobs = Vec::new();
2485        let mut vote_pos: Vec<usize> = Vec::new();
2486        for &j in &recovered {
2487            let seat_key = format!("judge-{}", j + 1);
2488            let spec = self.roles.judges[j].clone();
2489            let seat = self.seat(&seat_key, &spec.id);
2490            let mut text = prompt::final_vote(&labels, &language);
2491            if !has_context(&spec, &seat, sessions) {
2492                text = format!(
2493                    "{}\n\n# Candidates\n\n{}",
2494                    text,
2495                    self.candidate_block(&candidates, &base_short)
2496                );
2497            }
2498            vote_jobs.push(SeatJob {
2499                spec,
2500                seat,
2501                prompt: text,
2502                cwd: root.join(seat_key),
2503                timeout,
2504                allow_write: false,
2505                sessions,
2506                artifacts: artifacts.clone(),
2507                stem: format!("vote-judge-{}-recover", j + 1),
2508            });
2509            vote_pos.push(j);
2510        }
2511        let allowed = labels.clone();
2512        let mut vote_losses = Vec::new();
2513        let vote_retries = self.state.config.graph.retries;
2514        let vote_cache = self.state.config.cache_dir();
2515        let ctx = WaveCtx {
2516            run: &run_id,
2517            node: "vote",
2518            prompts: &prompts,
2519            cache: vote_cache.as_deref(),
2520        };
2521        let votes = ask_json_wave::<FinalVote>(
2522            vote_jobs,
2523            Arc::clone(&self.sem),
2524            vote_retries,
2525            &ctx,
2526            &mut vote_losses,
2527            &mut self.state,
2528            &move |v: &FinalVote| match v.label() {
2529                Some(c) if allowed.contains(&c) => Ok(()),
2530                other => bail!("vote {other:?} is not one of {allowed:?}"),
2531            },
2532        )
2533        .await;
2534        for (&j, (seat, res)) in vote_pos.iter().zip(votes) {
2535            let agent_id = seat.agent.clone();
2536            self.state.seats.insert(seat.key.clone(), seat);
2537            match res {
2538                Ok((v, _)) => {
2539                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
2540                        rec.vote = v.label();
2541                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2542                    } else {
2543                        self.state.votes.push(VoteRecord {
2544                            judge: j + 1,
2545                            agent: agent_id,
2546                            vote: v.label(),
2547                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
2548                            changed: false,
2549                        });
2550                    }
2551                    self.state.event(
2552                        "recover",
2553                        format!("judge {} voted again after the limit", j + 1),
2554                    );
2555                }
2556                Err(e) => {
2557                    self.state
2558                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
2559                }
2560            }
2561        }
2562
2563        // A seat that ranked again is present even if its re-vote failed —
2564        // `tally` falls back to the initial ranking's first choice — so clear
2565        // its quota loss. Seats that still fail keep theirs and stay absent.
2566        if !recovered.is_empty() {
2567            let recovered_keys: BTreeSet<String> = recovered
2568                .iter()
2569                .map(|&j| format!("judge-{}", j + 1))
2570                .collect();
2571            self.state
2572                .quota
2573                .retain(|q| !recovered_keys.contains(&q.seat));
2574        }
2575
2576        // Recompute the verdict from the refreshed panel.
2577        self.state.tally = None;
2578        self.tally()?;
2579        Ok(self
2580            .state
2581            .tally
2582            .as_ref()
2583            .map(|t| t.met_quorum)
2584            .unwrap_or(false))
2585    }
2586
2587    // ----------------------------------------------------------------- fold
2588
2589    async fn fold_losers(&mut self) -> Result<()> {
2590        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
2591            return Ok(());
2592        };
2593        let repo = self.state.repo.clone();
2594        let mut folded = Vec::new();
2595        for i in 0..self.state.candidates.len() {
2596            let c = &self.state.candidates[i];
2597            if c.label == winner || c.folded {
2598                continue;
2599            }
2600            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
2601            git::worktree_remove(&repo, &wt).await.ok();
2602            git::branch_delete(&repo, &branch).await.ok();
2603            self.state.candidates[i].folded = true;
2604            folded.push(label.to_string());
2605        }
2606        // The judges are finished; their checkouts are pure cost from here.
2607        let root = self.state.worktree_root();
2608        for j in 1..=self.roles.judges.len() {
2609            let wt = root.join(format!("judge-{j}"));
2610            if wt.exists() {
2611                git::worktree_remove(&repo, &wt).await.ok();
2612            }
2613        }
2614        // The design-deliberation stage is finished by the time a tally
2615        // exists — same reasoning as the judges above.
2616        if self.state.config.graph.advise {
2617            for k in 1..=self.state.config.graph.advisors {
2618                let wt = root.join(format!("advisor-{k}"));
2619                if wt.exists() {
2620                    git::worktree_remove(&repo, &wt).await.ok();
2621                }
2622            }
2623        }
2624        if !folded.is_empty() {
2625            self.state
2626                .event("fold", format!("folded candidates {}", folded.join(", ")));
2627            self.state.save()?;
2628        }
2629        Ok(())
2630    }
2631
2632    // ------------------------------------------------------------ base sync
2633
2634    /// Land the winner's tree on the current tip of `<remote>/<base>` before
2635    /// anything verifies it.
2636    ///
2637    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
2638    /// read whatever is checked out in the winner's worktree. Left alone that
2639    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
2640    /// it when the run *branched* - and a run takes long enough that the base
2641    /// has usually moved by the time it gets here. A gate that ran there
2642    /// answers "green on the commit this run started from", not "green on
2643    /// what is about to land", and the difference showed up three times in
2644    /// one day as a green run whose merge would have reverted a file another
2645    /// pull request had already landed.
2646    ///
2647    /// Reuses [`git::rebase_branch_in_temp`] rather than a second
2648    /// implementation of the same idea: `land::Step::Rebase` already worked
2649    /// out the rules - throwaway worktree, conflict stops and reports rather
2650    /// than feeding a fixer, nothing runs in the primary tree - and a second
2651    /// rebase path is exactly the kind of drift `resolve_base`'s own doc
2652    /// warns about ("two answers to a question nobody notices until a diff is
2653    /// wrong").
2654    ///
2655    /// Bounded by [`BASE_SYNC_ROUNDS`], counted in `state.base_sync.attempts`
2656    /// so it survives a park/resume. A conflict or a push failure sets
2657    /// `state.base_sync.conflict` and leaves the branch and worktree exactly
2658    /// as they were - untouched, for a person to look at - which is also what
2659    /// makes re-entering this function afterwards a no-op instead of a second
2660    /// attempt at the same wall.
2661    async fn sync_to_base(&mut self) -> Result<()> {
2662        if self
2663            .state
2664            .base_sync
2665            .as_ref()
2666            .is_some_and(|s| s.conflict.is_some())
2667        {
2668            return Ok(());
2669        }
2670        let Some(winner) = self.state.winner().cloned() else {
2671            return Ok(());
2672        };
2673
2674        let repo = self.state.repo.clone();
2675        let remote = self.state.config.merge.remote.clone();
2676        let base_branch = self.state.base_branch.clone();
2677        let tracking = format!("{remote}/{base_branch}");
2678
2679        git::fetch(&repo, &remote, &base_branch).await.ok();
2680        // No network, or the remote never had this branch: `resolve_base`
2681        // already treats that as non-fatal at branch time, and a run that got
2682        // this far must not be blocked by it here either.
2683        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
2684            return Ok(());
2685        };
2686
2687        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
2688        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
2689        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
2690
2691        if behind == 0 {
2692            self.state.base_sync = Some(BaseSync {
2693                tip,
2694                behind: 0,
2695                attempts,
2696                conflict: None,
2697            });
2698            self.state.save()?;
2699            return Ok(());
2700        }
2701
2702        if attempts >= BASE_SYNC_ROUNDS {
2703            let why = format!(
2704                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
2705                 rebase(s); rebasing again would only race it",
2706                winner.branch
2707            );
2708            self.state.status = RunStatus::Blocked;
2709            self.state.base_sync = Some(BaseSync {
2710                tip,
2711                behind,
2712                attempts,
2713                conflict: Some(why.clone()),
2714            });
2715            self.state.event("land", why);
2716            self.state.save()?;
2717            return Ok(());
2718        }
2719
2720        self.state.event(
2721            "land",
2722            format!(
2723                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
2724                winner.branch
2725            ),
2726        );
2727        self.state.save()?;
2728
2729        let scratch = self.state.dir().join("base-sync");
2730        let rebased = git::rebase_branch_in_temp(&repo, &scratch, &winner.branch, &tracking).await;
2731        let attempts = attempts + 1;
2732        match rebased {
2733            Ok(None) => {
2734                // The branch ref moved, but a worktree that already had it
2735                // checked out (the winner's) was not told; sync its index and
2736                // files before anything reads them.
2737                git::sync_to_head(&winner.worktree).await?;
2738                self.state.base_sync = Some(BaseSync {
2739                    tip: tip.clone(),
2740                    behind: 0,
2741                    attempts,
2742                    conflict: None,
2743                });
2744                self.state
2745                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
2746            }
2747            Ok(Some(conflict)) => {
2748                let why = format!(
2749                    "{} conflicts with {tracking} and did not rebase: {}",
2750                    winner.branch,
2751                    conflict.chars().take(600).collect::<String>()
2752                );
2753                self.state.status = RunStatus::Blocked;
2754                self.state.base_sync = Some(BaseSync {
2755                    tip,
2756                    behind,
2757                    attempts,
2758                    conflict: Some(why.clone()),
2759                });
2760                self.state.event("land", why);
2761            }
2762            Err(e) => {
2763                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
2764                self.state.status = RunStatus::Blocked;
2765                self.state.base_sync = Some(BaseSync {
2766                    tip,
2767                    behind,
2768                    attempts,
2769                    conflict: Some(why.clone()),
2770                });
2771                self.state.event("land", why);
2772            }
2773        }
2774        self.state.save()?;
2775        Ok(())
2776    }
2777
2778    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
2779    /// last landed the winner on, once it has run, else the commit the run
2780    /// branched from.
2781    ///
2782    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
2783    /// and `vote` all happen before there is a winner to rebase, so they
2784    /// compare every candidate against the branch point on purpose, and a
2785    /// base that moves after they are already done cannot change an answer
2786    /// they already gave.
2787    fn landing_base(&self) -> String {
2788        self.state
2789            .base_sync
2790            .as_ref()
2791            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
2792    }
2793
2794    // --------------------------------------------------------------- review
2795
2796    async fn review_loop(&mut self) -> Result<()> {
2797        // A base that would not rebase is a person's decision, not a review
2798        // round: nothing here would change the answer, and reviewers and a
2799        // fixer would be spending real budget on a tree that cannot land
2800        // regardless of what they find.
2801        if self
2802            .state
2803            .base_sync
2804            .as_ref()
2805            .is_some_and(|s| s.conflict.is_some())
2806        {
2807            return Ok(());
2808        }
2809        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2810        // agent files with `magi task add` name the run that paid for it. The
2811        // prompt overlay is cloned alongside it because the waves borrow it
2812        // while `self` is mutably borrowed by the node's own bookkeeping.
2813        let run_id = self.state.id.clone();
2814        let prompts = self.state.config.prompts.clone();
2815        let Some(winner) = self.state.winner().cloned() else {
2816            return Ok(());
2817        };
2818        let max_rounds = self.state.config.graph.review_rounds;
2819        // A clean round, an exhausted round budget, or a stalled tree (see
2820        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
2821        // they are recorded — recomputed here, not read off `status`, so a
2822        // reentry into a run that already stopped restates the identical
2823        // verdict instead of silently handing back whatever an earlier node
2824        // in this same walk clobbered `status` to (a solo-candidate
2825        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
2826        // below runs an empty range once the budget is spent, and would
2827        // otherwise fall through without touching `status` at all.
2828        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
2829            self.state.status = status;
2830            self.state.save()?;
2831            return Ok(());
2832        }
2833        self.state.status = RunStatus::Reviewing;
2834
2835        let repo = self.state.repo.clone();
2836        let root = self.state.worktree_root();
2837        let language = self.state.config.graph.language.clone();
2838        let sessions = self.state.config.graph.sessions;
2839        let artifacts = agent::artifacts_dir(&self.state.dir());
2840        let base = self.landing_base();
2841        let base_short = short(&base);
2842        let reviewers = self.roles.reviewers.clone();
2843        let shell = self.state.config.shell();
2844
2845        let mut prev_e2e: Option<String> = None;
2846        for round in (self.state.reviews.len() + 1)..=max_rounds {
2847            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
2848            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
2849            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
2850
2851            // Each reviewer gets its own detached checkout of exactly this
2852            // commit: nobody can perturb the winner's tree, and the fixer can
2853            // keep working without racing a reviewer.
2854            let mut jobs = Vec::new();
2855            for (r, spec) in reviewers.iter().cloned().enumerate() {
2856                let wt = root.join(format!("review-{}", r + 1));
2857                if wt.exists() {
2858                    git::reset_detached(&wt, &head).await?;
2859                } else {
2860                    git::worktree_add_detached(&repo, &wt, &head).await?;
2861                }
2862                let seat_key = format!("review-{}", r + 1);
2863                let seat = self.seat(&seat_key, &spec.id);
2864                jobs.push(SeatJob {
2865                    prompt: prompt::review(&prompt::ReviewCtx {
2866                        instruction: &self.state.instruction,
2867                        branch: &winner.branch,
2868                        base_short: &base_short,
2869                        stat: &stat,
2870                        patch: &patch,
2871                        e2e: prev_e2e.as_deref(),
2872                        reviewers: reviewers.len(),
2873                        round,
2874                        rounds: max_rounds,
2875                        // A review-only run has no rankings, so nothing
2876                        // competed for this patch and the reviewer is told so.
2877                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
2878                        lens: Lens::for_seat(r),
2879                        language: &language,
2880                    }),
2881                    spec,
2882                    seat,
2883                    cwd: wt,
2884                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
2885                    allow_write: false,
2886                    sessions,
2887                    artifacts: artifacts.clone(),
2888                    stem: format!("review-{round}-{}", r + 1),
2889                });
2890            }
2891
2892            self.state.event(
2893                "review",
2894                format!(
2895                    "round {round}: {} reviewers on {}",
2896                    jobs.len(),
2897                    short(&head)
2898                ),
2899            );
2900            let mut quota_losses = Vec::new();
2901            let review_retries = self.state.config.graph.retries;
2902            let review_cache = self.state.config.cache_dir();
2903            let ctx = WaveCtx {
2904                run: &run_id,
2905                node: "review",
2906                prompts: &prompts,
2907                cache: review_cache.as_deref(),
2908            };
2909            let results = ask_json_wave::<Review>(
2910                jobs,
2911                Arc::clone(&self.sem),
2912                review_retries,
2913                &ctx,
2914                &mut quota_losses,
2915                &mut self.state,
2916                &|_: &Review| Ok(()),
2917            )
2918            .await;
2919            // Counted before the move below: how many of *this* round's
2920            // reviewer seats were lost to their own rate limit, as opposed to
2921            // a crash, a timeout, or unparsable output — see `round_is_clean`.
2922            let round_quota_missing = quota_losses.len();
2923            self.state.quota.extend(quota_losses);
2924
2925            let mut records = Vec::new();
2926            let mut all_findings = Vec::new();
2927            for (r, (seat, res)) in results.into_iter().enumerate() {
2928                let agent_id = seat.agent.clone();
2929                self.state.seats.insert(seat.key.clone(), seat);
2930                let mut record = ReviewRecord {
2931                    reviewer: r + 1,
2932                    agent: agent_id,
2933                    summary: String::new(),
2934                    findings: Vec::new(),
2935                    vote: None,
2936                    failed: None,
2937                    duration_ms: 0,
2938                };
2939                match res {
2940                    Ok((review, out)) => {
2941                        // Sanitized here, at the point every other piece of
2942                        // agent prose in this file is (candidate summaries,
2943                        // deliberation turns, vote reasons): a reviewer's own
2944                        // words are the one thing about it that could name
2945                        // it, and reconsideration below broadcasts this same
2946                        // summary and these same findings to every other
2947                        // seat on the panel.
2948                        record.summary =
2949                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
2950                        record.vote = Some(review.vote);
2951                        record.duration_ms = out.duration_ms;
2952                        for (n, mut f) in review.findings.into_iter().enumerate() {
2953                            // ids are magi's, never the agent's: the fixer's
2954                            // adoption report is keyed by them.
2955                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
2956                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
2957                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
2958                            // `file` is agent-supplied prose too, never
2959                            // checked against the real tree — the same
2960                            // exposure `title`/`detail` above have, just in
2961                            // a field easy to forget because it looks like a
2962                            // path rather than free text.
2963                            f.file = f
2964                                .file
2965                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
2966                            all_findings.push(f.clone());
2967                            record.findings.push(f);
2968                        }
2969                        self.state.event(
2970                            "review",
2971                            format!(
2972                                "round {round}: reviewer {} voted {} with {} finding(s)",
2973                                r + 1,
2974                                review.vote.label(),
2975                                record.findings.len()
2976                            ),
2977                        );
2978                    }
2979                    Err(e) => {
2980                        record.failed = Some(e.to_string());
2981                        self.state.event(
2982                            "review",
2983                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
2984                        );
2985                    }
2986                }
2987                records.push(record);
2988            }
2989
2990            // Tally the round's votes and, if they split, spend the one
2991            // round of reconsideration the split -> deliberate -> revote
2992            // shape `judge`/`vote` use for the panel, sized down to what a
2993            // read-only review round can afford: one round, and a revote
2994            // rather than an argument, because the panel already wrote its
2995            // reasoning down as findings the first time around.
2996            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
2997            let vote_split =
2998                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
2999            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
3000            if vote_split {
3001                self.state.event(
3002                    "review",
3003                    format!(
3004                        "round {round}: votes split ({}) — one round of reconsideration",
3005                        initial_votes
3006                            .iter()
3007                            .map(|v| v.label())
3008                            .collect::<Vec<_>>()
3009                            .join(", ")
3010                    ),
3011                );
3012                // Seats read every seat's findings and votes, still numbered
3013                // and never named — the same anonymity `review` itself keeps.
3014                let panel: Vec<ReviewSeatReport<'_>> = records
3015                    .iter()
3016                    .filter_map(|r| {
3017                        r.vote.map(|vote| ReviewSeatReport {
3018                            reviewer: r.reviewer,
3019                            vote,
3020                            summary: &r.summary,
3021                            findings: &r.findings,
3022                        })
3023                    })
3024                    .collect();
3025
3026                let mut jobs = Vec::new();
3027                let mut seats_at = Vec::new();
3028                for (r, spec) in reviewers.iter().cloned().enumerate() {
3029                    // A seat with no initial vote has nothing to reconsider
3030                    // from and stays absent, the same as it stayed absent
3031                    // from `panel` above.
3032                    if records[r].vote.is_none() {
3033                        continue;
3034                    }
3035                    let wt = root.join(format!("review-{}", r + 1));
3036                    let seat_key = format!("review-{}", r + 1);
3037                    let seat = self.seat(&seat_key, &spec.id);
3038                    // A seat with no live session has already forgotten the
3039                    // initial review's prompt — restate the patch it is
3040                    // voting on, the same as `deliberate`/`vote` do for a
3041                    // judge in the same position.
3042                    let patch_ctx = if has_context(&spec, &seat, sessions) {
3043                        None
3044                    } else {
3045                        Some(ReviewPatch {
3046                            branch: &winner.branch,
3047                            base_short: &base_short,
3048                            stat: &stat,
3049                            patch: &patch,
3050                        })
3051                    };
3052                    let prompt = prompt::review_reconsider(&ReviewReconsiderCtx {
3053                        instruction: &self.state.instruction,
3054                        reviewer: r + 1,
3055                        lens: Lens::for_seat(r),
3056                        panel: &panel,
3057                        patch: patch_ctx,
3058                        round,
3059                        rounds: max_rounds,
3060                        language: &language,
3061                    });
3062                    jobs.push(SeatJob {
3063                        prompt,
3064                        spec,
3065                        seat,
3066                        cwd: wt,
3067                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
3068                        allow_write: false,
3069                        sessions,
3070                        artifacts: artifacts.clone(),
3071                        stem: format!("review-{round}-reconsider-{}", r + 1),
3072                    });
3073                    seats_at.push(r);
3074                }
3075
3076                let mut recon_quota_losses = Vec::new();
3077                let recon_cache = self.state.config.cache_dir();
3078                let recon_ctx = WaveCtx {
3079                    run: &run_id,
3080                    node: "review",
3081                    prompts: &prompts,
3082                    cache: recon_cache.as_deref(),
3083                };
3084                let recon_results = ask_json_wave::<ReviewRevote>(
3085                    jobs,
3086                    Arc::clone(&self.sem),
3087                    review_retries,
3088                    &recon_ctx,
3089                    &mut recon_quota_losses,
3090                    &mut self.state,
3091                    &|_: &ReviewRevote| Ok(()),
3092                )
3093                .await;
3094                self.state.quota.extend(recon_quota_losses);
3095
3096                for (&r, (seat, res)) in seats_at.iter().zip(recon_results) {
3097                    let agent_id = seat.agent.clone();
3098                    self.state.seats.insert(seat.key.clone(), seat);
3099                    let mut rec = ReviewRevoteRecord {
3100                        reviewer: r + 1,
3101                        agent: agent_id,
3102                        vote: None,
3103                        reason: String::new(),
3104                        failed: None,
3105                    };
3106                    match res {
3107                        Ok((rv, _)) => {
3108                            rec.vote = Some(rv.vote);
3109                            rec.reason =
3110                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
3111                            self.state.event(
3112                                "review",
3113                                format!(
3114                                    "round {round}: reviewer {} revoted {}",
3115                                    r + 1,
3116                                    rv.vote.label()
3117                                ),
3118                            );
3119                        }
3120                        Err(e) => {
3121                            rec.failed = Some(e.to_string());
3122                            self.state.event(
3123                                "review",
3124                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
3125                            );
3126                        }
3127                    }
3128                    reconsideration.push(rec);
3129                }
3130            } else if initial_votes.len() > 1 {
3131                self.state.event(
3132                    "review",
3133                    format!(
3134                        "round {round}: votes agreed ({}) — no reconsideration",
3135                        initial_votes[0].label()
3136                    ),
3137                );
3138            }
3139
3140            // The final vote per seat is its revote where reconsideration
3141            // ran and answered, its initial vote otherwise — the same
3142            // fallback `tally` uses for a judge whose private vote failed.
3143            let final_votes: Vec<ReviewVote> = records
3144                .iter()
3145                .filter_map(|r| {
3146                    reconsideration
3147                        .iter()
3148                        .find(|rv| rv.reviewer == r.reviewer)
3149                        .and_then(|rv| rv.vote)
3150                        .or(r.vote)
3151                })
3152                .collect();
3153            let round_verdict = ReviewVote::worst(final_votes);
3154
3155            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
3156            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
3157            // A round that already has a blocking finding and a round left to
3158            // try is going back to the fixer no matter what `verify.e2e`
3159            // says, so running it first only spends the loop's slowest step
3160            // (minutes, for a Rust repo's full test suite) on a head about
3161            // to be rewritten. Deferred, never skipped: `verify.e2e` still
3162            // runs once a round has no blocking findings left (see
3163            // `round_is_clean`, which a deferred — empty — `e2e` can never
3164            // satisfy since `blocking` is nonzero whenever this branch is
3165            // taken), and `stop_reviewing` forces a real run before it will
3166            // ever read a deferred round as green.
3167            let defer_e2e =
3168                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
3169            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
3170                let reason =
3171                    format!("{blocking} blocking finding(s) already required a fix this round");
3172                self.state.event(
3173                    "verify",
3174                    format!(
3175                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
3176                         {}); it will run once a round has none left",
3177                        short(&head)
3178                    ),
3179                );
3180                (Vec::new(), false, true, Some(reason))
3181            } else {
3182                let e2e_commands = self.state.config.verify.e2e.clone();
3183                let cache_dir = self.state.config.cache_dir();
3184                let context = format!("round {round}");
3185                let (e2e, verify_retried) = with_cache_lease(
3186                    &mut self.state,
3187                    cache_dir.as_deref(),
3188                    "e2e",
3189                    "e2e",
3190                    &winner.worktree,
3191                    &head,
3192                    verify_timeout,
3193                    &context,
3194                    |state, budget| {
3195                        let shell = shell.clone();
3196                        let e2e_commands = e2e_commands.clone();
3197                        let worktree = winner.worktree.clone();
3198                        let context = context.clone();
3199                        async move {
3200                            run_e2e_with_retry(
3201                                state,
3202                                &shell,
3203                                &e2e_commands,
3204                                &worktree,
3205                                budget,
3206                                &context,
3207                            )
3208                            .await
3209                        }
3210                    },
3211                )
3212                .await;
3213                (e2e, verify_retried, false, None)
3214            };
3215
3216            let e2e_failures: String = e2e
3217                .iter()
3218                .filter(|o| !o.ok())
3219                .map(|o| format!("$ {}\n{}\n", o.command, o.output_tail))
3220                .collect();
3221
3222            let expected = records.len();
3223            let answered = records.iter().filter(|r| r.failed.is_none()).count();
3224            let incomplete = answered < expected;
3225            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
3226            let policy = self.state.config.graph.incomplete_review;
3227            let clean = round_is_clean(
3228                blocking,
3229                e2e_ok,
3230                answered,
3231                expected,
3232                round_quota_missing,
3233                policy,
3234            );
3235
3236            let mut round_record = ReviewRound {
3237                round,
3238                head: head.clone(),
3239                verified_head: None,
3240                reviews: records,
3241                e2e,
3242                verify_retried,
3243                e2e_deferred,
3244                e2e_defer_reason,
3245                fix: None,
3246                blocking,
3247                answered,
3248                expected,
3249                clean,
3250                progressed: false,
3251                vote_split,
3252                reconsideration,
3253                verdict: round_verdict,
3254            };
3255
3256            if incomplete {
3257                let missing: Vec<String> = round_record
3258                    .reviews
3259                    .iter()
3260                    .filter(|r| r.failed.is_some())
3261                    .map(|r| format!("review-{}", r.reviewer))
3262                    .collect();
3263                self.state.event(
3264                    "review",
3265                    format!(
3266                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
3267                        missing.join(", ")
3268                    ),
3269                );
3270            }
3271
3272            if clean {
3273                self.state.event(
3274                    "review",
3275                    if incomplete && policy == IncompleteReviewPolicy::Warn {
3276                        format!(
3277                            "round {round}: clean (warn policy, incomplete panel) — no \
3278                             blocking findings from the seats that answered, verification green"
3279                        )
3280                    } else if incomplete {
3281                        format!(
3282                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
3283                             quorum) — no blocking findings from the seats that answered, \
3284                             verification green",
3285                            expected - answered
3286                        )
3287                    } else {
3288                        format!("round {round}: clean — no blocking findings, verification green")
3289                    },
3290                );
3291                self.state.reviews.push(round_record);
3292                self.state.status = RunStatus::Gating;
3293                self.state.save()?;
3294                return Ok(());
3295            }
3296
3297            // Nothing was raised and verification passed, but not every seat
3298            // answered and `round_is_clean` still refused to call it clean —
3299            // either a seat is missing for a reason other than its own quota
3300            // (a crash, a timeout, unparsable output — worth another try), or
3301            // every seat that could have answered lost its quota and nobody
3302            // is left to decide on: re-review rather than send the fixer
3303            // after a round with nothing to fix.
3304            if incomplete && blocking == 0 && e2e_ok {
3305                self.state.reviews.push(round_record);
3306                self.state.save()?;
3307                if round == max_rounds {
3308                    self.state.status = RunStatus::Blocked;
3309                    self.state.event(
3310                        "review",
3311                        format!(
3312                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
3313                             refusing to call it clean",
3314                            expected - answered
3315                        ),
3316                    );
3317                    return Ok(());
3318                }
3319                prev_e2e = None;
3320                continue;
3321            }
3322
3323            if round == max_rounds {
3324                self.state.reviews.push(round_record);
3325                return self
3326                    .stop_reviewing(
3327                        &format!(
3328                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
3329                        ),
3330                        &shell,
3331                        &winner.worktree,
3332                    )
3333                    .await;
3334            }
3335
3336            // Fix. The winner's own implementer seat continues its conversation:
3337            // the competition is over, so context is pure benefit now.
3338            let (fix_spec, fix_seat_key) = match &self.roles.fixer {
3339                Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3340                _ => (
3341                    self.state
3342                        .config
3343                        .agent(&winner.agent)
3344                        .cloned()
3345                        .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3346                    format!("impl-{}", winner.label),
3347                ),
3348            };
3349            let seat = self.seat(&fix_seat_key, &fix_spec.id);
3350            let blocking_findings: Vec<_> = all_findings
3351                .iter()
3352                .filter(|f| f.severity.blocks())
3353                .cloned()
3354                .collect();
3355            let job = SeatJob {
3356                prompt: prompt::fix(
3357                    &self.state.instruction,
3358                    &blocking_findings,
3359                    (!e2e_failures.is_empty()).then_some(e2e_failures.as_str()),
3360                    e2e_deferred,
3361                    round,
3362                    max_rounds,
3363                    &language,
3364                ),
3365                spec: fix_spec.clone(),
3366                seat,
3367                cwd: winner.worktree.clone(),
3368                timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
3369                allow_write: true,
3370                sessions,
3371                artifacts: artifacts.clone(),
3372                stem: format!("fix-{round}"),
3373            };
3374            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
3375            let cache = self.state.config.cache_dir();
3376            let ctx = WaveCtx {
3377                run: &run_id,
3378                node: "fix",
3379                prompts: &prompts,
3380                cache: cache.as_deref(),
3381            };
3382            let (seat, out) =
3383                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3384            let agent_id = seat.agent.clone();
3385
3386            let mut fix = FixRecord {
3387                agent: agent_id,
3388                addressed: Vec::new(),
3389                rejected: Vec::new(),
3390                notes: String::new(),
3391                committed: false,
3392                failed: None,
3393                duration_ms: 0,
3394                continuation: None,
3395            };
3396            let mut continuation = ContinuationRecord::not_needed();
3397            let mut final_seat = seat.clone();
3398            match out {
3399                AgentOutcome::Ok(o) => {
3400                    fix.duration_ms = o.duration_ms;
3401                    let parsed = verdict::extract_json::<FixReport>(&o.text);
3402                    // A parsed report standing next to a command this same
3403                    // reply's own CLI never confirmed the exit status of is
3404                    // not a resolved answer — the identical `CommandEvidence`
3405                    // `state.jobs` renders, read here instead of only on
3406                    // display, per the completion judgment and the shown
3407                    // record needing to agree.
3408                    let incomplete_reason = match &parsed {
3409                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
3410                            "the reply parsed, but it reported a command whose own CLI never \
3411                             confirmed an exit status"
3412                                .to_owned(),
3413                        ),
3414                        Ok(_) => None,
3415                        Err(e) => Some(e.to_string()),
3416                    };
3417                    match incomplete_reason {
3418                        None => {
3419                            let report = parsed.expect("checked Ok above");
3420                            fix.addressed = report.addressed;
3421                            fix.rejected = report.rejected;
3422                            fix.notes =
3423                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
3424                        }
3425                        Some(reason) => {
3426                            let (resumed_seat, resolved, failure, cont) = self
3427                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
3428                                .await;
3429                            fix.duration_ms += cont.cumulative_wait_ms;
3430                            continuation = cont;
3431                            final_seat = resumed_seat;
3432                            match resolved {
3433                                Some(report) => {
3434                                    fix.addressed = report.addressed;
3435                                    fix.rejected = report.rejected;
3436                                    fix.notes = blind::sanitize_prose(
3437                                        &report.notes,
3438                                        &self.state.config.blind,
3439                                    );
3440                                }
3441                                None => fix.failed = failure,
3442                            }
3443                        }
3444                    }
3445                }
3446                // The CLI's raw error JSON is not a fix report to parse.
3447                AgentOutcome::Dropped(o) => {
3448                    fix.duration_ms = o.duration_ms;
3449                    let why = o
3450                        .dropped
3451                        .as_ref()
3452                        .map(|d| d.why.as_str())
3453                        .unwrap_or("the CLI ended the stream without delivering its answer");
3454                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
3455                }
3456                AgentOutcome::Quota(o) => {
3457                    self.state.quota.push(QuotaLoss {
3458                        seat: final_seat.key.clone(),
3459                        node: "fix".to_owned(),
3460                        at: Timestamp::now(),
3461                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3462                    });
3463                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
3464                }
3465                AgentOutcome::Failed(e) => fix.failed = Some(e),
3466            }
3467            fix.continuation = Some(continuation);
3468            self.state.seats.insert(final_seat.key.clone(), final_seat);
3469            git::commit_all(
3470                &winner.worktree,
3471                &format!("magi: review round {round} fixes (uncommitted work)"),
3472            )
3473            .await
3474            .ok();
3475            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
3476            fix.committed = after != before;
3477            // Judged by what `git` says moved against base, never by the
3478            // fixer's own `addressed`/`rejected` count — see
3479            // `ReviewRound::progressed`. Propagated with `?`, the same as the
3480            // `patch` snapshot above: swallowing this error would default
3481            // `diff_after` to empty, which almost always differs from a
3482            // non-empty `patch` and reads as "progressed" — exactly backwards
3483            // for a `git` failure the stagnation check cannot see through.
3484            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
3485            let progressed = diff_after != patch;
3486            let commit_note = if fix.committed {
3487                "committed"
3488            } else {
3489                "NO new commit"
3490            };
3491            let tree_note = if progressed {
3492                "changed vs base"
3493            } else {
3494                "unchanged vs base"
3495            };
3496            self.state.event(
3497                "fix",
3498                match &fix.failed {
3499                    // Distinct on purpose from "0 addressed, 0 rejected": the
3500                    // fixer's own diff still landed (blocking counts do keep
3501                    // falling round over round), only its adoption report did
3502                    // not come back, so this must never read like every
3503                    // finding was reviewed and declined.
3504                    Some(reason) => {
3505                        format!(
3506                            "round {round}: fixer's adoption report was lost ({reason}); \
3507                             {commit_note}, tree {tree_note}"
3508                        )
3509                    }
3510                    None => format!(
3511                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
3512                         {tree_note}{}",
3513                        fix.addressed.len(),
3514                        fix.rejected.len(),
3515                        if continuation.outcome == ContinuationOutcome::Resumed {
3516                            format!(
3517                                " (adoption report recovered after {} continuation(s))",
3518                                continuation.attempts
3519                            )
3520                        } else {
3521                            String::new()
3522                        },
3523                    ),
3524                },
3525            );
3526            round_record.fix = Some(fix);
3527            round_record.progressed = progressed;
3528            self.state.reviews.push(round_record);
3529            self.state.save()?;
3530
3531            // The fixer's own report never came back this round, even after
3532            // `continue_fix_report`'s own budget was spent on it — not an
3533            // ordinary "no report" (dropped stream, quota, plain failure),
3534            // which already reads that way and is left to the existing round
3535            // budget. Stopping here, rather than opening another round, is
3536            // what keeps a next reviewer/fixer wave from ever being
3537            // dispatched onto `winner.worktree` while whatever the seat's
3538            // last call may still have running there is unaccounted for: no
3539            // process liveness check exists (and none is being added — see
3540            // AGENTS.md/this task's own scope), so the only way to honour
3541            // "nothing starts before a valid report returns" is to not start
3542            // anything further on this worktree from this run at all.
3543            if matches!(
3544                continuation.outcome,
3545                ContinuationOutcome::Exhausted
3546                    | ContinuationOutcome::QuotaLost
3547                    | ContinuationOutcome::NoSession
3548            ) {
3549                return self
3550                    .stop_reviewing(
3551                        "the fixer's adoption report never came back, even after resuming its \
3552                         own seat; refusing to start another round against the same worktree \
3553                         while that is unresolved",
3554                        &shell,
3555                        &winner.worktree,
3556                    )
3557                    .await;
3558            }
3559
3560            prev_e2e = (!e2e_failures.is_empty()).then_some(e2e_failures);
3561
3562            let streak = self
3563                .state
3564                .reviews
3565                .iter()
3566                .rev()
3567                .take_while(|r| !r.progressed)
3568                .count();
3569            if streak >= STAGNANT_LIMIT {
3570                return self
3571                    .stop_reviewing(
3572                        &format!(
3573                            "the tree has not moved against base for {streak} round(s) in a row"
3574                        ),
3575                        &shell,
3576                        &winner.worktree,
3577                    )
3578                    .await;
3579            }
3580        }
3581        Ok(())
3582    }
3583
3584    /// Decide, from the last recorded round's own verification, whether
3585    /// stopping the review loop is a hand-off or a genuine block.
3586    ///
3587    /// Called once the loop has given up trying — the round budget is spent,
3588    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
3589    /// findings still open, never while a round is still clean or the
3590    /// incomplete-panel case handled inline above. Gate and e2e are facts
3591    /// about the tree; a lingering review finding is an opinion, and this
3592    /// workload's own `magi stats` puts reviewer precision low enough
3593    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
3594    /// must not by itself stand between a green, verified change and the
3595    /// human who decides what to do with it. A red e2e is not an opinion, so
3596    /// that case still blocks, with the failing command and a tail of its
3597    /// output recorded here rather than left in `run.json` for someone to go
3598    /// find.
3599    ///
3600    /// A round that deferred its own e2e (see [`Config::graph`]'s
3601    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
3602    /// only because nothing ran, and treating an empty list as a passing one
3603    /// here is exactly the "deferred painted green" bug this function exists
3604    /// to not have. When the last round deferred, this makes the real run —
3605    /// on the actual worktree this loop is about to stop touching — before
3606    /// deciding anything.
3607    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
3608        let round_idx = self.state.reviews.len() - 1;
3609        let needs_catchup_run = {
3610            let last = &self.state.reviews[round_idx];
3611            last.e2e.is_empty() && last.e2e_deferred
3612        };
3613        if needs_catchup_run {
3614            let round = self.state.reviews[round_idx].round;
3615            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
3616            let commands = self.state.config.verify.e2e.clone();
3617            let verified_head = git::rev_parse(worktree, "HEAD").await?;
3618            let cache_dir = self.state.config.cache_dir();
3619            let context =
3620                format!("round {round}: deferred e2e, now catching up before the final decision");
3621            let (outcomes, verify_retried) = with_cache_lease(
3622                &mut self.state,
3623                cache_dir.as_deref(),
3624                "e2e",
3625                "e2e",
3626                worktree,
3627                &verified_head,
3628                timeout,
3629                &context,
3630                |state, budget| {
3631                    let shell = shell.to_vec();
3632                    let commands = commands.clone();
3633                    let context = context.clone();
3634                    async move {
3635                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
3636                            .await
3637                    }
3638                },
3639            )
3640            .await;
3641            // The catch-up run never actually happened - the shared build
3642            // cache could not be acquired or confirmed fresh in time (see
3643            // `CommandOutcome::resource_blocked`'s own doc) - so this round's
3644            // `e2e`/`e2e_deferred` are left exactly as they were:
3645            // `needs_catchup_run` above still reads true the next time this
3646            // is reached, and the round stays deferred rather than recording
3647            // contention as a red e2e and blocking the run on it.
3648            if verify_inconclusive(&outcomes) {
3649                self.state.save()?;
3650                return Ok(());
3651            }
3652            let last = &mut self.state.reviews[round_idx];
3653            last.e2e = outcomes;
3654            last.verify_retried = verify_retried;
3655            last.e2e_deferred = false;
3656            if verified_head != last.head {
3657                last.verified_head = Some(verified_head);
3658            }
3659        }
3660        let last = &self.state.reviews[round_idx];
3661        let red: Vec<String> = last
3662            .e2e
3663            .iter()
3664            .filter(|o| !o.ok())
3665            .map(|o| {
3666                format!(
3667                    "`{}` -> {:?}\n{}",
3668                    o.command,
3669                    o.code,
3670                    tail(&o.output_tail, EVENT_OUTPUT_TAIL)
3671                )
3672            })
3673            .collect();
3674        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
3675
3676        if red.is_empty() {
3677            self.state.event(
3678                "review",
3679                format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
3680            );
3681            self.state.status = RunStatus::Gating;
3682        } else {
3683            self.state
3684                .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
3685            self.state.status = RunStatus::Blocked;
3686        }
3687        self.state.save()?;
3688        Ok(())
3689    }
3690
3691    // ----------------------------------------------------------------- gate
3692
3693    async fn gate(&mut self) -> Result<()> {
3694        // Judged by the review record itself, not by `status`: a solo
3695        // candidate's `judge`/`deliberate` skip rewrites `status` on every
3696        // reentry (see `judge`), and trusting it here is exactly how a run
3697        // that exhausted its review budget got gated and merged a second
3698        // time around. `review_conclusion` recomputes the review loop's own
3699        // verdict from the round records themselves — `Gating` for a clean
3700        // round or a hand-off (see `stop_reviewing`), anything else means the
3701        // loop is still going or genuinely blocked.
3702        // A base the winner could not be replayed onto is a decision, not a
3703        // round: there is no landing tree to gate. Read as its own record for
3704        // the same reason the review verdict is.
3705        if self.state.status == RunStatus::Failed
3706            || self
3707                .state
3708                .base_sync
3709                .as_ref()
3710                .is_some_and(|s| s.conflict.is_some())
3711            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
3712                != Some(RunStatus::Gating)
3713        {
3714            return Ok(());
3715        }
3716        if self.state.gate_ran {
3717            // `review_loop` derives its conclusion from the clean review
3718            // record on every reentry and therefore puts a completed run back
3719            // in `Gating`. A recorded gate is a stronger, terminal fact:
3720            // retain its original command output (or lack of any, for a repo
3721            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
3722            // doc) and restore `Blocked` on a real failure rather than
3723            // pretending the command is still running or running it a second
3724            // time. `gate_ran == false` remains the only shape — unattempted,
3725            // or a resource-blocked retry — that may still need to execute a
3726            // command.
3727            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
3728                self.state.status = RunStatus::Blocked;
3729                self.state.save()?;
3730            }
3731            return Ok(());
3732        }
3733        let Some(winner) = self.state.winner().cloned() else {
3734            return Ok(());
3735        };
3736        self.state.status = RunStatus::Gating;
3737        let shell = self.state.config.shell();
3738        let gate_commands = self.state.config.verify.gate.clone();
3739        // Zero commands has nothing to run and nothing that could touch the
3740        // shared build cache, so it never needs a lease: `Config::cache_dir`
3741        // is derived from `verify.e2e` too, so a repo with no `verify.gate`
3742        // commands but a `CARGO_TARGET_DIR`-using `verify.e2e` would
3743        // otherwise queue behind an unrelated run's lease and come back
3744        // resource-blocked - `gate_ran` would stay false on nothing but
3745        // cache contention, for a step that had nothing to check in the
3746        // first place.
3747        let outcomes = if gate_commands.is_empty() {
3748            Vec::new()
3749        } else {
3750            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
3751            let cache_dir = self.state.config.cache_dir();
3752            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3753            let (outcomes, _) = with_cache_lease(
3754                &mut self.state,
3755                cache_dir.as_deref(),
3756                "gate",
3757                "gate",
3758                &winner.worktree,
3759                &head,
3760                timeout,
3761                "final gate",
3762                |_state, budget| {
3763                    let shell = shell.clone();
3764                    let gate_commands = gate_commands.clone();
3765                    let worktree = winner.worktree.clone();
3766                    async move {
3767                        let (outcomes, timed_out_pids) =
3768                            run_commands(&shell, &gate_commands, &worktree, budget).await;
3769                        (outcomes, false, timed_out_pids)
3770                    }
3771                },
3772            )
3773            .await;
3774            outcomes
3775        };
3776        if outcomes.is_empty() {
3777            // Nothing configured to check — distinct from every other
3778            // silence in this run's event log, since an empty `gate` alone
3779            // no longer says whether the gate ran at all (see
3780            // `RunState::gate_ran`'s own doc).
3781            self.state.event(
3782                "gate",
3783                "no gate commands configured; nothing to check, passing",
3784            );
3785        }
3786        for o in &outcomes {
3787            self.state.event(
3788                "gate",
3789                format!(
3790                    "`{}` -> {}",
3791                    o.command,
3792                    if o.ok() {
3793                        "pass".to_owned()
3794                    } else {
3795                        format!(
3796                            "FAIL ({:?})\n{}",
3797                            o.code,
3798                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
3799                        )
3800                    }
3801                ),
3802            );
3803        }
3804        // A resource-blocked outcome means the gate command never actually
3805        // ran - the shared build cache could not be acquired or confirmed
3806        // fresh in time - which is evidence about the machine, not about the
3807        // tree (see `CommandOutcome::resource_blocked`'s own doc). Recording
3808        // it as a red gate would mark a run `Blocked` on nothing but
3809        // contention magi has already logged above; leaving `self.state.gate`
3810        // empty and `self.state.gate_ran` false instead keeps the shape this
3811        // function already treats as "still needs to run" (see the
3812        // early-return above), so the next call retries the command rather
3813        // than concluding anything.
3814        if verify_inconclusive(&outcomes) {
3815            self.state.save()?;
3816            return Ok(());
3817        }
3818        let passed = outcomes.iter().all(CommandOutcome::ok);
3819        self.state.gate = outcomes;
3820        self.state.gate_ran = true;
3821        if !passed {
3822            self.state.status = RunStatus::Blocked;
3823            self.state.event("gate", "gate failed; not merging");
3824        }
3825        self.state.save()?;
3826        Ok(())
3827    }
3828
3829    // ---------------------------------------------------------------- merge
3830
3831    async fn merge(&mut self) -> Result<()> {
3832        // Same reasoning as `gate`: ask the review and gate records directly
3833        // rather than `status`, which a solo-candidate `judge`/`deliberate`
3834        // skip can rewrite on reentry to something that no longer says
3835        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
3836        // so a hand-off (open findings, green verification) reaches merge
3837        // exactly like a genuinely clean round does.
3838        //
3839        // A run resumed mid-`land` never reaches here at all: `execute`
3840        // recognises `RunStatus::Landing` before it even calls `prep`, and
3841        // routes straight to `run_land` instead. That has to happen a level
3842        // up from this function, not with a check in here, because
3843        // `review_loop`'s own status recomputation (see its doc) runs
3844        // *before* `merge` on every reentry and would otherwise overwrite
3845        // the `Landing` marker with `Gating` before this node ever saw it.
3846        if self
3847            .state
3848            .base_sync
3849            .as_ref()
3850            .is_some_and(|s| s.conflict.is_some())
3851            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
3852                != Some(RunStatus::Gating)
3853            // `gate_ran == false` is not "passed" - `gate` leaves it false
3854            // both before it has ever run and when its last attempt was
3855            // resource-blocked (see `Runner::gate`'s own doc), and neither is
3856            // permission to merge on nothing but the review record. Only a
3857            // gate that actually ran - zero commands configured and
3858            // vacuously passed, or one or more that all exited 0 - may
3859            // proceed; `RunState::gate_status` is the single place that
3860            // reading is computed.
3861            || !self.state.gate_status().ok()
3862        {
3863            return Ok(());
3864        }
3865        // This node's own record, not `status`: `status == Ready` is not
3866        // unique to the harmless `MergeMode::None` path this line was
3867        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
3868        // run's PR was closed without merging — and on that run `mode` is
3869        // still `Pr`, so a reentry that fell through here would push and
3870        // open a second pull request. `self.state.merge` is set exactly once
3871        // this node (or `land`) has already produced a verdict, under every
3872        // mode, which is what "already done" actually means here.
3873        if self.state.merge.is_some() {
3874            return Ok(());
3875        }
3876        let Some(winner) = self.state.winner().cloned() else {
3877            return Ok(());
3878        };
3879        let repo = self.state.repo.clone();
3880        let base = self.state.base_branch.clone();
3881        let mode = self.state.config.merge.mode;
3882        let style = self.state.config.merge.style;
3883        let message = pr_body(&self.state, winner.label);
3884
3885        let outcome = match mode {
3886            MergeMode::None => MergeOutcome {
3887                mode,
3888                ok: true,
3889                detail: manual_merge_command(style, &repo, &winner.branch, &message),
3890            },
3891            MergeMode::Local => {
3892                let on = git::current_branch(&repo).await?;
3893                if on.as_deref() != Some(base.as_str()) {
3894                    MergeOutcome {
3895                        mode,
3896                        ok: false,
3897                        detail: format!(
3898                            "{} has {} checked out, not the base branch {base}",
3899                            repo.display(),
3900                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
3901                        ),
3902                    }
3903                } else if !git::is_clean(&repo).await? {
3904                    MergeOutcome {
3905                        mode,
3906                        ok: false,
3907                        detail: format!("{} is dirty; refusing to merge", repo.display()),
3908                    }
3909                } else {
3910                    let out = match style {
3911                        MergeStyle::Merge => {
3912                            git::merge_no_ff(&repo, &winner.branch, &message).await?
3913                        }
3914                        MergeStyle::Squash => {
3915                            git::merge_squash(&repo, &winner.branch, &message).await?
3916                        }
3917                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
3918                    };
3919                    MergeOutcome {
3920                        mode,
3921                        ok: out.ok(),
3922                        detail: if out.ok() { out.stdout } else { out.stderr },
3923                    }
3924                }
3925            }
3926            MergeMode::Pr => {
3927                let remote = self.state.config.merge.remote.clone();
3928                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
3929                if !pushed.ok() {
3930                    MergeOutcome {
3931                        mode,
3932                        ok: false,
3933                        detail: pushed.stderr,
3934                    }
3935                } else {
3936                    let out = gh_pr_create(&winner.worktree, &base, &winner.branch, &message).await;
3937                    match out {
3938                        Ok(url) => MergeOutcome {
3939                            mode,
3940                            ok: true,
3941                            detail: url,
3942                        },
3943                        Err(e) => MergeOutcome {
3944                            mode,
3945                            ok: false,
3946                            detail: e.to_string(),
3947                        },
3948                    }
3949                }
3950            }
3951        };
3952
3953        self.state.status = match (mode, outcome.ok) {
3954            (MergeMode::None, _) => RunStatus::Ready,
3955            (_, true) => RunStatus::Merged,
3956            (_, false) => RunStatus::Blocked,
3957        };
3958        self.state.event(
3959            "merge",
3960            format!(
3961                "{:?}: {}",
3962                mode,
3963                outcome.detail.lines().next().unwrap_or("")
3964            ),
3965        );
3966        self.state.merge = Some(outcome);
3967        self.state.save()?;
3968
3969        // The PR is open and the run would historically stop here, leaving the
3970        // operator to watch checks, feed review comments back to a fixer, and
3971        // merge. That was done by hand six times in one session before this
3972        // existed. Opt-in, because merging is the one irreversible thing magi
3973        // can do to a repository.
3974        if self.state.config.graph.land
3975            && mode == MergeMode::Pr
3976            && self.state.status == RunStatus::Merged
3977        {
3978            self.run_land().await?;
3979        }
3980        // `run_land` may have left `status` at `Landing` - still waiting on
3981        // CI or the owner's approval, not actually settled - so this has to
3982        // read whatever `status` ended up as here, not the `Merged` this
3983        // function set a few lines up.
3984        self.settle_questions();
3985        Ok(())
3986    }
3987
3988    /// Enter `land`.
3989    ///
3990    /// Shared between a fresh run's first pass through [`Runner::merge`] and
3991    /// a resumed run's re-entry. `land::land` itself is what serialises the
3992    /// two git-mutating moments inside the loop — the rebase push and
3993    /// `gh pr merge` — per repository (see its own doc); nothing here needs
3994    /// to hold a lock across the whole call, and doing so would serialise
3995    /// this run's CI wait against a *different* run's land-approval resume
3996    /// in the same repository, which is exactly the "must not wait on
3997    /// another task" property the daemon's slot-freeing exists to give.
3998    async fn run_land(&mut self) -> Result<()> {
3999        let url = self
4000            .state
4001            .merge
4002            .as_ref()
4003            .map(|m| m.detail.clone())
4004            .unwrap_or_default();
4005        let url = url.lines().next().unwrap_or("").trim().to_owned();
4006        if !url.starts_with("http") {
4007            return Ok(());
4008        }
4009        // A land failure is not a lost run: the work is on a branch and the
4010        // pull request is open, which is exactly where a human takes over.
4011        match land::land(&mut self.state, &url).await {
4012            Ok(pr) if self.state.parked => {
4013                // `land` already saved the parked marker; nothing here
4014                // overrides `status` back to a terminal value while an
4015                // approval is still outstanding.
4016                let _ = pr;
4017            }
4018            Ok(pr) => {
4019                self.state.status = match pr.state {
4020                    land::PrLifecycle::Merged => RunStatus::Merged,
4021                    _ => RunStatus::Blocked,
4022                };
4023                // Downstream of a confirmed merge only - see
4024                // `bump::should_release_bump`'s own doc for why this one
4025                // check covers all three of `land`'s success paths.
4026                // Best-effort: the run already landed, so a failure here
4027                // (the decision call, `gh`, `cargo`) is recorded and never
4028                // turns a landed run into a failed one.
4029                if bump::should_release_bump(self.state.status)
4030                    && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
4031                {
4032                    self.state
4033                        .event("bump", format!("release bump skipped: {e:#}"));
4034                }
4035                self.state.save()?;
4036            }
4037            Err(e) => {
4038                self.state.status = RunStatus::Blocked;
4039                self.state.event("land", format!("gave up: {e}"));
4040                self.state.save()?;
4041            }
4042        }
4043        Ok(())
4044    }
4045
4046    // -------------------------------------------------------------- helpers
4047
4048    /// Fetch or create a seat, keeping its conversation across nodes.
4049    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
4050        if let Some(existing) = self.state.seats.get(key)
4051            && existing.agent == agent
4052        {
4053            return existing.clone();
4054        }
4055        let fresh = SeatState::new(key, agent, self.state.seed);
4056        self.state.seats.insert(key.to_owned(), fresh.clone());
4057        fresh
4058    }
4059
4060    /// A candidate rendered for judging, with the leak policy applied.
4061    fn view(&self, c: &Candidate) -> CandidateView {
4062        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4063            .unwrap_or_default();
4064        let (patch, _) = blind::sanitize_patch(
4065            &format!("candidate {} patch", c.label),
4066            &raw,
4067            &self.state.config.blind,
4068        );
4069        CandidateView {
4070            label: c.label,
4071            branch: c.branch.clone(),
4072            summary: c.summary.clone(),
4073            stat: c.stat.clone(),
4074            patch,
4075        }
4076    }
4077
4078    /// The full candidate set as prompt text, for seats with no live session.
4079    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
4080        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
4081        prompt::judge(
4082            "(see above)",
4083            &views,
4084            self.roles.judges.len(),
4085            base_short,
4086            "en",
4087        )
4088    }
4089
4090    /// Anonymised transcript for judge `self_idx`.
4091    ///
4092    /// The initial rankings are always the opening statements. Seeding them
4093    /// only when no turn had been taken yet meant every judge after the first
4094    /// argued against a single voice instead of against the actual split — the
4095    /// disagreement is the information, so it is always on the table.
4096    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
4097        let mut turns = Vec::new();
4098        for j in &self.state.judgements {
4099            if j.ranking.is_empty() {
4100                continue;
4101            }
4102            let reasons = j
4103                .reasons
4104                .iter()
4105                .map(|(k, v)| format!("- {k}: {v}"))
4106                .collect::<Vec<_>>()
4107                .join("\n");
4108            turns.push(Turn {
4109                who: format!("Judge {} (opening ranking)", j.judge),
4110                is_self: j.judge == self_idx + 1,
4111                body: format!(
4112                    "Ranked {}{}{reasons}",
4113                    j.ranking.iter().collect::<String>(),
4114                    if reasons.is_empty() {
4115                        ""
4116                    } else {
4117                        ", because:\n"
4118                    }
4119                ),
4120            });
4121        }
4122        for t in self
4123            .state
4124            .deliberation
4125            .iter()
4126            .flat_map(|r| r.turns.iter())
4127            .chain(current)
4128        {
4129            turns.push(Turn {
4130                who: format!("Judge {}", t.judge),
4131                is_self: t.judge == self_idx + 1,
4132                body: t.body.clone(),
4133            });
4134        }
4135        turns
4136    }
4137}
4138
4139/// Does this seat still hold the context a follow-up prompt would rely on?
4140fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
4141    agent::has_session(spec.kind, seat, sessions)
4142}
4143
4144/// Did this reply report running a command whose own CLI never confirmed an
4145/// exit status?
4146///
4147/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
4148/// command *finished* (see that type's own doc), so this can only be `true`
4149/// for a command whose completion event carried no readable exit code — not
4150/// for one that simply is not mentioned at all. That is the one signal this
4151/// crate can read, from the same record `state.jobs` renders, about a reply
4152/// standing next to work its own CLI cannot vouch for finishing; it is
4153/// deliberately not a check on the exit code's *value* (a fixer legitimately
4154/// runs a command that fails mid-iteration before it succeeds) and not a
4155/// guess at a command still running in the background (which emits no event
4156/// at all, and so leaves no evidence here to find).
4157fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
4158    commands.iter().any(|c| c.exit_code.is_none())
4159}
4160
4161fn short(commit: &str) -> String {
4162    commit.chars().take(7).collect()
4163}
4164
4165fn make_executable(path: &Path) -> Result<()> {
4166    #[cfg(unix)]
4167    {
4168        use std::os::unix::fs::PermissionsExt as _;
4169        let mut perms = std::fs::metadata(path)?.permissions();
4170        perms.set_mode(0o755);
4171        std::fs::set_permissions(path, perms)?;
4172    }
4173    #[cfg(not(unix))]
4174    {
4175        let _ = path;
4176    }
4177    Ok(())
4178}
4179
4180/// What every seat in one batch shares: where the answers are attributed, the
4181/// prompt overlay they inherit, and the build cache they are told to use.
4182///
4183/// A struct rather than four more parameters: `wave` also needs the run's
4184/// state (to record who is answering right now) and the attempt number, and
4185/// eight positional arguments is both unreadable and a clippy error.
4186struct WaveCtx<'a> {
4187    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
4188    /// paid for it.
4189    run: &'a str,
4190    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
4191    node: &'a str,
4192    prompts: &'a Prompts,
4193    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
4194    cache: Option<&'a Path>,
4195}
4196
4197/// Run one job, honouring the parallelism budget.
4198async fn run_one(
4199    job: SeatJob,
4200    sem: Arc<Semaphore>,
4201    ctx: &WaveCtx<'_>,
4202    state: &mut RunState,
4203    attempt: usize,
4204) -> (SeatState, AgentOutcome) {
4205    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
4206        .await
4207        .pop()
4208        .expect("one job in, one result out");
4209    (seat, out)
4210}
4211
4212/// Run every job concurrently, capped by the semaphore, preserving order.
4213///
4214/// Every seat in the batch is recorded into [`RunState::active`] before the
4215/// wave starts and cleared as each answer lands, so the run's own record says
4216/// who is still being waited on rather than only who finished.
4217async fn wave(
4218    jobs: Vec<SeatJob>,
4219    sem: Arc<Semaphore>,
4220    ctx: &WaveCtx<'_>,
4221    state: &mut RunState,
4222    attempt: usize,
4223) -> Vec<(usize, SeatState, AgentOutcome)> {
4224    let WaveCtx {
4225        run,
4226        node,
4227        prompts,
4228        cache,
4229    } = *ctx;
4230    for job in &jobs {
4231        state.seat_started(node, &job.seat.key, job.timeout, attempt);
4232    }
4233    if let Err(e) = state.save() {
4234        // A failed persist of "who is answering right now" must not abort the
4235        // wave: the seats are already being asked, and the alternative is
4236        // losing the answers to save a status line nobody may even be
4237        // watching.
4238        tracing::warn!("could not persist in-progress seats: {e:#}");
4239    }
4240    // Hold the shared build cache's lease for the whole batch, not per job:
4241    // several candidates (an implement wave) or a fixer legitimately share
4242    // one cache concurrently within this run, and that stays untouched — a
4243    // single lease taken once for the whole wave and released once it is
4244    // done is what stops a *different* borrower (another run's own wave, its
4245    // e2e/gate, a human's `magi review`) from interleaving a build into the
4246    // same directory while this one is in flight. Best-effort, not
4247    // all-or-nothing: a wave that cannot get the lease within its own
4248    // longest job's budget still runs — an hour of paid implementer calls is
4249    // not thrown away over cache contention — but every write-allowed seat
4250    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
4251    // below), the same fallback a read-only seat always gets, rather than
4252    // building into a directory this run was never granted. The identity
4253    // record is still invalidated below either way, so the next tracked
4254    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
4255    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
4256    let wait_started = Instant::now();
4257    let cache_guard = if let Some(cache_dir) = cache {
4258        if jobs_had_a_writer {
4259            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
4260            let budget = jobs
4261                .iter()
4262                .map(|j| j.timeout)
4263                .max()
4264                .unwrap_or(Duration::from_secs(60));
4265            acquire_cache_lease(state, cache_dir, &owner, budget, node)
4266                .await
4267                .ok()
4268        } else {
4269            None
4270        }
4271    } else {
4272        None
4273    };
4274    // Carved out of each job's own budget, not added on top of it: a seat
4275    // that waited behind the lease must not also get its full timeout
4276    // afterward, or a run contended on the cache could double the time it
4277    // spends per wave. `saturating_sub` floors at zero rather than
4278    // wrapping - a job whose whole budget was spent waiting starts with
4279    // none left, which is the honest number, not a free minimum.
4280    let waited_for_lease = wait_started.elapsed();
4281    let mut set = tokio::task::JoinSet::new();
4282    let overlay = prompts.overlay(node);
4283    for (i, mut job) in jobs.into_iter().enumerate() {
4284        job.timeout = job.timeout.saturating_sub(waited_for_lease);
4285        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
4286        if cache.is_some() {
4287            job.prompt.push('\n');
4288            job.prompt
4289                .push_str(&prompt::build_cache_note(node, job.allow_write));
4290        }
4291        let sem = Arc::clone(&sem);
4292        let run = run.to_owned();
4293        let node = node.to_owned();
4294        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
4295        // `prompt::build_cache_note`'s doc for why setting it anyway is
4296        // exactly how a sandboxed reviewer's write refusal got reported as a
4297        // defect in the patch, not a property of its own seat. And a
4298        // write-allowed one is handed it only when the lease above was
4299        // actually acquired: a wave that could not get it (`cache_guard` is
4300        // `None`, see its own comment) must not send seats to build into a
4301        // directory this run does not hold - that is the exact concurrent,
4302        // unmanaged-write race this module exists to prevent, not something
4303        // "proceeding anyway" is allowed to reintroduce.
4304        let cache = cache
4305            .filter(|_| job.allow_write && cache_guard.is_some())
4306            .map(Path::to_path_buf);
4307        set.spawn(async move {
4308            let _permit = sem.acquire().await;
4309            let mut seat = job.seat;
4310            let out = agent::invoke(
4311                &job.spec,
4312                &mut seat,
4313                &Invocation {
4314                    cwd: &job.cwd,
4315                    prompt: &job.prompt,
4316                    timeout: job.timeout,
4317                    allow_write: job.allow_write,
4318                    sessions: job.sessions,
4319                    artifacts: &job.artifacts,
4320                    stem: &job.stem,
4321                    run: &run,
4322                    node: &node,
4323                    cache_dir: cache.as_deref(),
4324                    attachments: &[],
4325                },
4326            )
4327            .await;
4328            let out = match out {
4329                Ok(o) if o.usable() => AgentOutcome::Ok(o),
4330                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
4331                // Billed work the CLI failed to hand over is not an ordinary
4332                // failure, but its text is the CLI's raw error JSON, not an
4333                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
4334                // read it as one by forgetting to check. `usable()` is always
4335                // false here (dropped implies an empty response), so this has
4336                // to be checked before the catch-all `Failed` below or the
4337                // one shape this exists for is lost with the rest.
4338                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
4339                Ok(o) if o.timed_out => AgentOutcome::Failed("timed out".to_owned()),
4340                Ok(o) => AgentOutcome::Failed(format!(
4341                    "exited with {:?} and no usable output",
4342                    o.exit_code
4343                )),
4344                Err(e) => AgentOutcome::Failed(e.to_string()),
4345            };
4346            (i, seat, out)
4347        });
4348    }
4349    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
4350    while let Some(joined) = set.join_next().await {
4351        let (i, seat, out) = match joined {
4352            Ok(v) => v,
4353            // No seat to clear: a panicked task never reported which one it
4354            // was. The defensive sweep below this loop is what stops that
4355            // seat's `active` entry from surviving forever.
4356            Err(e) => {
4357                tracing::error!("agent task panicked: {e}");
4358                continue;
4359            }
4360        };
4361        state.seat_finished(&seat.key);
4362        record_jobs(state, node, &seat.key, &out);
4363        if let Err(e) = state.save() {
4364            tracing::warn!("could not persist a seat's completion: {e:#}");
4365        }
4366        if collected.len() <= i {
4367            collected.resize_with(i + 1, || None);
4368        }
4369        collected[i] = Some((i, seat, out));
4370    }
4371    // Belt-and-braces for the panic branch above: every seat this exact batch
4372    // started shares this `(node, attempt)` pair, and every seat that finished
4373    // normally already cleared itself, so anything left tagged with it here
4374    // can only be a panicked task's leftover. Cleared unconditionally rather
4375    // than left to read as still answering forever.
4376    if state
4377        .active
4378        .values()
4379        .any(|a| a.node == node && a.attempt == attempt)
4380    {
4381        state
4382            .active
4383            .retain(|_, a| !(a.node == node && a.attempt == attempt));
4384        if let Err(e) = state.save() {
4385            tracing::warn!("could not persist the end of a wave: {e:#}");
4386        }
4387    }
4388    // Whether or not the lease above was actually held, several worktrees
4389    // may just have built into the cache with nothing here able to name one
4390    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
4391    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
4392    // clean it might not have strictly needed; trusting a stale match would
4393    // cost it a wrong answer.
4394    if let Some(cache_dir) = cache
4395        && jobs_had_a_writer
4396    {
4397        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
4398    }
4399    if let Some(guard) = cache_guard {
4400        guard.release();
4401    }
4402    collected.into_iter().flatten().collect()
4403}
4404
4405/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
4406/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
4407/// this is data collection, not the fix-specific completion contract in
4408/// [`Runner::continue_fix_report`], and applies regardless of which node
4409/// asked.
4410///
4411/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
4412/// evidence from; `Failed` does not, and correctly contributes nothing — a
4413/// timeout or crash is not itself evidence about a command the seat may have
4414/// started.
4415fn record_jobs(state: &mut RunState, node: &str, seat: &str, out: &AgentOutcome) {
4416    let commands: &[agent::CommandEvidence] = match out {
4417        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
4418        AgentOutcome::Failed(_) => &[],
4419    };
4420    let checked_at = Timestamp::now();
4421    for c in commands {
4422        state.jobs.push(JobRecord {
4423            node: node.to_owned(),
4424            seat: seat.to_owned(),
4425            id: c.id.clone(),
4426            description: c.description.clone(),
4427            checked_at,
4428            status: match c.exit_code {
4429                Some(0) => JobStatus::Completed,
4430                Some(_) => JobStatus::Failed,
4431                None => JobStatus::Unknown,
4432            },
4433            exit_code: c.exit_code,
4434            result_summary: c.result_summary.clone(),
4435            source: c.source.clone(),
4436        });
4437    }
4438}
4439
4440/// Is a review round clean, given how many reviewer seats answered against
4441/// how many the round expected?
4442///
4443/// A seat that never answered (timeout, crash, unparsable output) is not a
4444/// seat that read the patch and found nothing — treating it as such is
4445/// exactly the bug this function exists to close. Under the default `block`
4446/// policy a missing seat can never be clean; `warn` still requires the seats
4447/// that *did* answer to have found nothing blocking and verification to be
4448/// green.
4449///
4450/// `quota_missing` narrows that `block` default for exactly one cause of
4451/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
4452/// a session limit lifts by the very next round buys nothing — the seat is
4453/// asked again with the same quota — so once every missing seat is accounted
4454/// for by a quota loss (and at least one seat *did* answer, so a decision has
4455/// something to rest on) the round is decided on the panel that could answer,
4456/// same as `warn` would. A panel that lost every seat to quota is not
4457/// decided here: `answered == 0` falls through to the existing `block`
4458/// fallback so a fully collapsed panel still waits rather than landing on no
4459/// review at all.
4460fn round_is_clean(
4461    blocking: usize,
4462    e2e_ok: bool,
4463    answered: usize,
4464    expected: usize,
4465    quota_missing: usize,
4466    policy: IncompleteReviewPolicy,
4467) -> bool {
4468    if blocking != 0 || !e2e_ok {
4469        return false;
4470    }
4471    if answered == expected || policy == IncompleteReviewPolicy::Warn {
4472        return true;
4473    }
4474    answered > 0 && expected - answered <= quota_missing
4475}
4476
4477/// The review loop's own conclusion, derived entirely from its persisted
4478/// round records and the round budget that produced them — never from
4479/// `status`, so a reentry (or `gate`/`merge` reading it independently)
4480/// recomputes the identical answer regardless of what an earlier node in the
4481/// same walk, or a previous walk, did to `status`.
4482///
4483/// `None` while more rounds remain to try, including when review never ran
4484/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
4485/// gone clean, or the budget is spent, or the tree has stopped moving (see
4486/// [`STAGNANT_LIMIT`]), the answer is one of two things:
4487///
4488/// - An incomplete panel that raised nothing is missing input, not a
4489///   verified tree — never a hand-off candidate, whatever verification said
4490///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
4491/// - Otherwise, green e2e on the last round hands off (see
4492///   [`Runner::stop_reviewing`]); red e2e blocks.
4493fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
4494    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
4495        return Some(RunStatus::Gating);
4496    }
4497    let last = reviews.last()?;
4498    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
4499    if reviews.len() < max_rounds && !stagnant {
4500        return None;
4501    }
4502    Some(if last.incomplete() && last.blocking == 0 {
4503        RunStatus::Blocked
4504    } else if last.e2e.iter().all(CommandOutcome::ok) {
4505        RunStatus::Gating
4506    } else {
4507        RunStatus::Blocked
4508    })
4509}
4510
4511/// How long a re-ask may take, given the budget the first attempt had.
4512///
4513/// A `nudged` retry is a request to restate an answer the seat has already
4514/// worked out: it carries no new work, so it does not deserve the original
4515/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
4516/// 133 seconds while a third sat for over ten minutes on a resumed session
4517/// holding 410 KB of prior output - and because the retry had inherited the
4518/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
4519/// judging round whose other seats were long finished.
4520///
4521/// A quarter of the budget, with a floor so that a deliberately short timeout
4522/// does not collapse to nothing. A retry that re-sends the whole prompt
4523/// (because the seat kept no context) is the original job again, and keeps the
4524/// original budget.
4525fn retry_budget(full: Duration, nudged: bool) -> Duration {
4526    if nudged {
4527        (full / 4).max(Duration::from_secs(120)).min(full)
4528    } else {
4529        full
4530    }
4531}
4532
4533/// Run a wave and parse each reply, re-asking the seats whose reply was
4534/// unusable.
4535///
4536/// The re-ask is a nudge rather than the whole prompt again when the seat still
4537/// holds its conversation, which is the difference between a cheap retry and
4538/// paying for the entire candidate set twice.
4539///
4540/// A seat that hits a rate limit is **not** re-asked: the same call will fail
4541/// the same way until the limit resets, so spending a retry attempt on it is
4542/// pure waste. Its loss is recorded in `losses` and it is returned as a failure
4543/// like any other absent seat — the caller decides whether the panel still has
4544/// a quorum.
4545#[allow(clippy::too_many_arguments)]
4546async fn ask_json_wave<T>(
4547    jobs: Vec<SeatJob>,
4548    sem: Arc<Semaphore>,
4549    retries: usize,
4550    ctx: &WaveCtx<'_>,
4551    losses: &mut Vec<QuotaLoss>,
4552    state: &mut RunState,
4553    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
4554) -> Vec<(SeatState, Result<(T, AgentOutput)>)>
4555where
4556    T: serde::de::DeserializeOwned + Send + 'static,
4557{
4558    let n = jobs.len();
4559    let originals: Vec<SeatJob> = jobs;
4560    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
4561    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
4562    let mut pending: Vec<usize> = (0..n).collect();
4563
4564    for attempt in 0..=retries {
4565        if pending.is_empty() {
4566            break;
4567        }
4568        let mut batch = Vec::with_capacity(pending.len());
4569        for &i in &pending {
4570            let src = &originals[i];
4571            // The prompt and the budget are one decision: a nudge restates
4572            // finished work, a re-sent prompt redoes it.
4573            let (prompt, timeout) = if attempt == 0 {
4574                (src.prompt.clone(), src.timeout)
4575            } else {
4576                let why = done[i]
4577                    .as_ref()
4578                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
4579                    .unwrap_or_else(|| "no parsable answer".to_owned());
4580                let nudge = prompt::nudge(&why);
4581                let nudged = has_context(&src.spec, &seats[i], src.sessions);
4582                let prompt = if nudged {
4583                    nudge
4584                } else {
4585                    format!("{}\n\n---\n\n{}", src.prompt, nudge)
4586                };
4587                (prompt, retry_budget(src.timeout, nudged))
4588            };
4589            batch.push(SeatJob {
4590                spec: src.spec.clone(),
4591                seat: seats[i].clone(),
4592                cwd: src.cwd.clone(),
4593                prompt,
4594                timeout,
4595                allow_write: src.allow_write,
4596                sessions: src.sessions,
4597                artifacts: src.artifacts.clone(),
4598                stem: if attempt == 0 {
4599                    src.stem.clone()
4600                } else {
4601                    format!("{}-retry{attempt}", src.stem)
4602                },
4603            });
4604        }
4605
4606        if attempt > 0 {
4607            let seats_out: Vec<&str> = pending
4608                .iter()
4609                .map(|&i| originals[i].seat.key.as_str())
4610                .collect();
4611            state.event(
4612                ctx.node,
4613                format!("retry {attempt}: re-asking {}", seats_out.join(", ")),
4614            );
4615        }
4616        let results = wave(batch, Arc::clone(&sem), ctx, state, attempt).await;
4617        let mut still = Vec::new();
4618        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
4619            seats[i] = seat;
4620            let (parsed, quota) = match out {
4621                AgentOutcome::Ok(o) => (
4622                    match verdict::extract_json::<T>(&o.text) {
4623                        Ok(v) => match validate(&v) {
4624                            Ok(()) => Ok((v, o)),
4625                            Err(e) => Err(e),
4626                        },
4627                        Err(e) => Err(e),
4628                    },
4629                    false,
4630                ),
4631                AgentOutcome::Quota(o) => {
4632                    losses.push(QuotaLoss {
4633                        seat: originals[i].seat.key.clone(),
4634                        node: ctx.node.to_owned(),
4635                        at: Timestamp::now(),
4636                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4637                    });
4638                    (
4639                        Err(anyhow::anyhow!("rate limited (quota); not retrying now")),
4640                        true,
4641                    )
4642                }
4643                // Not a parseable answer, but also not worth a special-cased
4644                // retry here: the nudge loop above already re-asks anything
4645                // that fails to parse, which is exactly what a dropped stream
4646                // needs. Just don't hand its raw error JSON to `extract_json`.
4647                AgentOutcome::Dropped(o) => {
4648                    let why = o
4649                        .dropped
4650                        .as_ref()
4651                        .map(|d| d.why.as_str())
4652                        .unwrap_or("the CLI ended the stream without delivering its answer");
4653                    (
4654                        Err(anyhow::anyhow!("the CLI dropped the stream ({why})")),
4655                        false,
4656                    )
4657                }
4658                AgentOutcome::Failed(e) => (Err(anyhow::anyhow!(e)), false),
4659            };
4660            let failed = parsed.is_err();
4661            done[i] = Some(parsed);
4662            // Do not re-ask a rate-limited seat (quota) — a retry is known to
4663            // fail the same way; and never re-ask a seat that already parsed.
4664            if failed && !quota {
4665                still.push(i);
4666            }
4667        }
4668        pending = still;
4669    }
4670
4671    seats
4672        .into_iter()
4673        .zip(done)
4674        .map(|(seat, res)| {
4675            (
4676                seat,
4677                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
4678            )
4679        })
4680        .collect()
4681}
4682
4683/// Acquire the shared build cache's lease, waiting out contention within
4684/// `budget` (never past it — see AGENTS.md's build-cache section on why an
4685/// unbounded wait is never acceptable).
4686///
4687/// A first, non-blocking check happens before ever waiting; if it finds the
4688/// lease busy, that fact is logged as a `verify` event *and* flushed with
4689/// [`RunState::save`] immediately — not only once the wait finally succeeds
4690/// or gives up — so a `magi show` run by a different process while this one
4691/// is still waiting reads a `run.json` that says so, rather than whatever it
4692/// looked like before the wait started. The same applies to the terminal
4693/// failure: logged and saved before this returns `Err`, so a caller that
4694/// could not get the lease at all still leaves a legible record of why.
4695async fn acquire_cache_lease(
4696    state: &mut RunState,
4697    cache_dir: &Path,
4698    owner: &crate::cache::Owner,
4699    budget: Duration,
4700    context: &str,
4701) -> Result<crate::cache::Guard> {
4702    let home = crate::run::home();
4703    let started = Instant::now();
4704    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
4705        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(g),
4706        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
4707        Err(e) => {
4708            state.event(
4709                "verify",
4710                format!("{context}: could not check the shared build cache: {e:#}"),
4711            );
4712            if let Err(e2) = state.save() {
4713                tracing::warn!("could not persist a cache-check failure: {e2:#}");
4714            }
4715            return Err(e);
4716        }
4717    };
4718    state.event(
4719        "verify",
4720        format!(
4721            "{context}: waiting for the shared build cache at {} ({})",
4722            cache_dir.display(),
4723            busy.describe()
4724        ),
4725    );
4726    if let Err(e) = state.save() {
4727        tracing::warn!("could not persist a cache wait: {e:#}");
4728    }
4729    let remaining = budget.saturating_sub(started.elapsed());
4730    match crate::cache::wait_for(&home, cache_dir, owner, remaining, Duration::from_secs(5)).await {
4731        Ok(g) => Ok(g),
4732        Err(e) => {
4733            state.event("verify", format!("{context}: {e:#}"));
4734            if let Err(e2) = state.save() {
4735                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
4736            }
4737            Err(e)
4738        }
4739    }
4740}
4741
4742/// Run `body` — a verify command batch — while holding the shared build
4743/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
4744/// never interleave with another borrower's build against the same
4745/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
4746/// timeout, or a human's own `magi review`. See the `cache` module doc for
4747/// why this matters more than Cargo's own per-target locking covers — two
4748/// *different* worktrees building the same package name/version into one
4749/// cache directory is a staleness bug, not a lock contention one.
4750///
4751/// The wait for the lease is carved out of `budget`, never on top of it —
4752/// `body` is handed whatever is left, so a caller's own node timeout is the
4753/// only clock involved, exactly what AGENTS.md's build-cache section asks
4754/// for ("never an unbounded wait"). When `cache_dir` is `None` — no shared
4755/// cache configured at all — this is a pass-through: `body` runs with the
4756/// full budget and nothing is leased.
4757///
4758/// A lease that cannot be acquired within `budget` is reported as a single
4759/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
4760/// verification — the same shape a spawn failure already takes in
4761/// [`run_commands`], so a caller need not special-case it.
4762#[allow(clippy::too_many_arguments)]
4763async fn with_cache_lease<'s, F, Fut>(
4764    state: &'s mut RunState,
4765    cache_dir: Option<&Path>,
4766    node: &str,
4767    seat: &str,
4768    worktree: &Path,
4769    head: &str,
4770    budget: Duration,
4771    context: &str,
4772    body: F,
4773) -> (Vec<CommandOutcome>, bool)
4774where
4775    F: FnOnce(&'s mut RunState, Duration) -> Fut,
4776    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
4777{
4778    let Some(cache_dir) = cache_dir else {
4779        let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
4780        return (outcomes, retried);
4781    };
4782    let home = crate::run::home();
4783    let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
4784    let started = Instant::now();
4785    let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context).await {
4786        Ok(g) => g,
4787        Err(e) => {
4788            return (
4789                vec![CommandOutcome {
4790                    command: "(waiting for the shared build cache)".to_owned(),
4791                    code: None,
4792                    output_tail: e.to_string(),
4793                    duration_ms: started.elapsed().as_millis() as u64,
4794                    resource_blocked: true,
4795                }],
4796                false,
4797            );
4798        }
4799    };
4800    let identity = crate::cache::Identity::new(worktree, head);
4801    if let Err(e) = crate::cache::ensure_fresh(&home, cache_dir, &identity) {
4802        // A failed freshness check means this process cannot vouch for what
4803        // is sitting in the cache right now - on Windows this is exactly the
4804        // "a stale test executable is still locked, `cargo clean -p` cannot
4805        // remove it" case the evidence log records. Running verify anyway
4806        // and reporting whatever it says would let a result nobody can trust
4807        // stand for the tree it claims to have checked; fail the step
4808        // instead of the patch.
4809        state.event(
4810            "verify",
4811            format!(
4812                "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
4813                worktree.display(),
4814                short(head)
4815            ),
4816        );
4817        guard.release();
4818        return (
4819            vec![CommandOutcome {
4820                command: "(confirming the shared build cache is fresh)".to_owned(),
4821                code: None,
4822                output_tail: e.to_string(),
4823                duration_ms: started.elapsed().as_millis() as u64,
4824                resource_blocked: true,
4825            }],
4826            false,
4827        );
4828    }
4829    let remaining = budget.saturating_sub(started.elapsed());
4830    let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
4831    // A timed-out command's process was only *asked* to die (`kill_on_drop`,
4832    // `start_kill`); confirm it actually has before handing the directory to
4833    // the next acquirer. See `wait_for_timed_out_children_to_die`'s own doc
4834    // for what this can and cannot see.
4835    if !timed_out_pids.is_empty() {
4836        wait_for_timed_out_children_to_die(&timed_out_pids).await;
4837    }
4838    guard.release();
4839    (outcomes, retried)
4840}
4841
4842/// Poll `pids` — commands [`run_commands`] reports as still running when its
4843/// own timeout elapsed — until every one is confirmed gone, or
4844/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
4845///
4846/// Real confirmation where confirmation is possible, not a substitute for
4847/// full process-tree observation: a grandchild the timed-out process spawned
4848/// and that survives independently of it is invisible to a pid check the
4849/// same way it always was, and continuing to observe and collect *that*
4850/// stays a different piece of work with its own owner. This only narrows a
4851/// fixed blind wait into an actual check of the pids this process does know
4852/// about.
4853async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
4854    wait_for_pids_with(
4855        pids,
4856        crate::proc::pid_alive,
4857        LEASE_RELEASE_POLL,
4858        LEASE_RELEASE_MAX_WAIT,
4859    )
4860    .await;
4861}
4862
4863/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
4864/// interval and ceiling supplied by the caller, so the polling *logic* -
4865/// returns as soon as every pid reports dead, gives up at the ceiling
4866/// otherwise - is testable on millisecond durations without asking the real
4867/// OS about a pid at all.
4868async fn wait_for_pids_with<F: Fn(u32) -> bool>(
4869    pids: &[u32],
4870    alive: F,
4871    poll: Duration,
4872    max_wait: Duration,
4873) {
4874    let deadline = Instant::now() + max_wait;
4875    loop {
4876        if pids.iter().all(|&pid| !alive(pid)) {
4877            return;
4878        }
4879        if Instant::now() >= deadline {
4880            return;
4881        }
4882        tokio::time::sleep(poll).await;
4883    }
4884}
4885
4886/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
4887/// admission that it could not even get a verify command to run, as opposed
4888/// to evidence the command actually produced? A caller that would otherwise
4889/// read a resource-blocked outcome as a red command must check this first:
4890/// see [`Runner::gate`], which retries rather than records `Blocked` when
4891/// this is true.
4892fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
4893    outcomes.iter().any(|o| o.resource_blocked)
4894}
4895
4896/// Describe one verify command's outcome for the event log, distinguishing a
4897/// build/link failure — the toolchain never produced a binary to run — from
4898/// an actual test failure, since only the latter is a verdict on the patch.
4899fn e2e_outcome_label(o: &CommandOutcome) -> String {
4900    if o.ok() {
4901        return "pass".to_owned();
4902    }
4903    let reason = if o.build_failed() {
4904        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
4905    } else {
4906        format!("FAIL ({:?})", o.code)
4907    };
4908    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
4909}
4910
4911/// Run `verify.e2e`, retrying once if the first attempt could not build or
4912/// link — a build/link failure is frequently a race against a shared
4913/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
4914/// `verify` event per command, tagged with `context` (normally `"round N"`)
4915/// so the two call sites that need this — the ordinary per-round leg in
4916/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
4917/// it will ever call a round green — read identically in the event log.
4918async fn run_e2e_with_retry(
4919    state: &mut RunState,
4920    shell: &[String],
4921    commands: &[String],
4922    worktree: &Path,
4923    timeout: Duration,
4924    context: &str,
4925) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
4926    let (mut e2e, mut timed_out_pids) = run_commands(shell, commands, worktree, timeout).await;
4927    for o in &e2e {
4928        state.event(
4929            "verify",
4930            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
4931        );
4932    }
4933    // A build/link failure is not a verdict on the patch — it is frequently a
4934    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
4935    // one retry before letting a red like that decide the round.
4936    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
4937    if verify_retried {
4938        state.event(
4939            "verify",
4940            format!(
4941                "{context}: verify could not build/link, not a test result — retrying once \
4942                 before concluding"
4943            ),
4944        );
4945        let retried = run_commands(shell, commands, worktree, timeout).await;
4946        e2e = retried.0;
4947        // Both attempts' timeouts matter, not just the last one: the first
4948        // attempt's descendants may still be alive alongside the retry's.
4949        timed_out_pids.extend(retried.1);
4950        for o in &e2e {
4951            state.event(
4952                "verify",
4953                format!(
4954                    "{context}: retry `{}` -> {}",
4955                    o.command,
4956                    e2e_outcome_label(o)
4957                ),
4958            );
4959        }
4960    }
4961    (e2e, verify_retried, timed_out_pids)
4962}
4963
4964/// Run configured shell commands in `cwd`, in order. The second element is
4965/// the pid of every command that hit `timeout` and was still running when
4966/// this stopped waiting on it (best-effort: `None` when the platform did not
4967/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
4968/// that releases a shared resource afterward needs to know.
4969async fn run_commands(
4970    shell: &[String],
4971    commands: &[String],
4972    cwd: &Path,
4973    timeout: Duration,
4974) -> (Vec<CommandOutcome>, Vec<u32>) {
4975    let mut out = Vec::new();
4976    let mut timed_out_pids = Vec::new();
4977    for command in commands {
4978        let started = Instant::now();
4979        let mut cmd = tokio::process::Command::new(&shell[0]);
4980        cmd.quiet();
4981        cmd.args(&shell[1..])
4982            .arg(command)
4983            .current_dir(cwd)
4984            .stdin(std::process::Stdio::null())
4985            .stdout(std::process::Stdio::piped())
4986            .stderr(std::process::Stdio::piped())
4987            .kill_on_drop(true);
4988        let spawned = cmd.spawn();
4989        let (code, body) = match spawned {
4990            Ok(child) => {
4991                // Captured before the child is consumed below: `kill_on_drop`
4992                // only *asks* the process to die when the timeout branch
4993                // drops it, and the pid is the only way anyone downstream can
4994                // later check whether that request actually took.
4995                let pid = child.id();
4996                match tokio::time::timeout(timeout, child.wait_with_output()).await {
4997                    Ok(Ok(o)) => {
4998                        let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
4999                        body.push_str(&String::from_utf8_lossy(&o.stderr));
5000                        (o.status.code(), body)
5001                    }
5002                    Ok(Err(e)) => (None, format!("failed to run: {e}")),
5003                    Err(_) => {
5004                        if let Some(pid) = pid {
5005                            timed_out_pids.push(pid);
5006                        }
5007                        (None, format!("timed out after {}s", timeout.as_secs()))
5008                    }
5009                }
5010            }
5011            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
5012        };
5013        out.push(CommandOutcome {
5014            command: command.clone(),
5015            code,
5016            output_tail: tail(&body, OUTPUT_TAIL),
5017            duration_ms: started.elapsed().as_millis() as u64,
5018            resource_blocked: false,
5019        });
5020    }
5021    (out, timed_out_pids)
5022}
5023
5024/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
5025/// section (`report::run`) and in the `merge` event this node records — for
5026/// the operator to run by hand.
5027///
5028/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
5029/// branch whose ruleset forbids merge commits (GitHub's "must not contain
5030/// merge commits", or "require linear history") rejects the push a `--no-ff`
5031/// merge would produce, which is exactly the guidance this function replaces.
5032/// `message`'s first line becomes the squash commit's subject, matching the
5033/// note `report::run` prints alongside this command — see that function for
5034/// why an explicit subject is not optional there.
5035fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
5036    let repo = repo.display();
5037    match style {
5038        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
5039        MergeStyle::Squash => {
5040            let subject = message.lines().next().unwrap_or(branch);
5041            format!(
5042                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
5043            )
5044        }
5045        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
5046    }
5047}
5048
5049/// The merge commit / pull request body: the task, and — when the winning
5050/// review round was not clean — the findings still open and whatever the
5051/// fixer declined, so `merge = "pr"` hands the reader the same material
5052/// `magi show` does rather than a pull request that reads clean while
5053/// `run.json` disagrees.
5054///
5055/// The first line doubles as the squash/merge commit subject
5056/// (`manual_merge_command`), which takes it via `message.lines().next()`
5057/// verbatim — so it has to be the task's own opening line, not run/candidate
5058/// bookkeeping. The pull request title (`gh_pr_create`) starts from the same
5059/// line but is further reshaped and truncated by `pr_title` to stay inside
5060/// GitHub's limit; see that function for why. "Merge magi run ec12 (candidate
5061/// B)" told a reader nothing about what landed once the run id had scrolled
5062/// off the PR list. That bookkeeping still needs to be findable, just not
5063/// from the title: the branch name already carries it
5064/// (`RunState::branch_for`), and the footer below repeats it as plain tags
5065/// for a reader holding only the merged commit or the PR body.
5066///
5067/// `state.instruction` can open with blank lines — a `--file` task is passed
5068/// through verbatim (`task_text` only rejects a body that is blank
5069/// *entirely*) — and `.lines().next()` on those reads back as `Some("")`, not
5070/// `None`. `trim_start` drops exactly those leading blank lines so the first
5071/// line is the task's real opening line, and the empty-after-trim case (a
5072/// whitespace-only instruction) falls back the same way `queue::title_from`
5073/// does for the same situation.
5074fn pr_body(state: &RunState, winner: char) -> String {
5075    let instruction = state.instruction.trim_start();
5076    let mut message = if instruction.is_empty() {
5077        "(empty task)".to_owned()
5078    } else {
5079        instruction.to_owned()
5080    };
5081
5082    let open = state.open_findings();
5083    if !open.is_empty() {
5084        message.push_str("\n\n## Open review findings\n\n");
5085        for f in &open {
5086            message.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
5087        }
5088    }
5089
5090    if let Some(fix) = state.reviews.last().and_then(|r| r.fix.as_ref())
5091        && !fix.rejected.is_empty()
5092    {
5093        message.push_str("\n## Declined by the fixer\n\n");
5094        for r in &fix.rejected {
5095            message.push_str(&format!("- `{}`: {}\n", r.id, r.why));
5096        }
5097    }
5098
5099    message.push_str(&format!(
5100        "\n\n---\nmagi:run/{} magi:candidate-{}\n",
5101        state.id,
5102        winner.to_ascii_lowercase()
5103    ));
5104
5105    message
5106}
5107
5108/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
5109/// under the hood, rejects a `title` over 256 characters and the whole
5110/// command fails — no PR at all, for a run whose body was otherwise fine
5111/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
5112/// that limit: `title_from` counts `chars()` (Unicode scalars), which is not
5113/// always how GitHub counts, plus one character for the trailing ellipsis
5114/// `title_from` may add. It is a margin, not a guarantee — a title packed
5115/// with multi-unit characters could still in principle land close to the
5116/// edge, but a real task title's occasional emoji or accented letter fits
5117/// comfortably inside it.
5118const PR_TITLE_MAX: usize = 240;
5119
5120/// The pull request title: the PR body's first line, reshaped and truncated
5121/// by [`queue::title_from`] the same way `magi show`'s task list titles are,
5122/// so it stays inside GitHub's limit on `--title` (see [`PR_TITLE_MAX`]).
5123fn pr_title(body: &str) -> String {
5124    queue::title_from(body, PR_TITLE_MAX)
5125}
5126
5127/// `gh pr create`, returning the PR url.
5128async fn gh_pr_create(cwd: &Path, base: &str, head: &str, body: &str) -> Result<String> {
5129    let title = pr_title(body);
5130    let out = tokio::process::Command::new("gh")
5131        .args([
5132            "pr", "create", "--base", base, "--head", head, "--title", &title, "--body", body,
5133        ])
5134        .current_dir(cwd)
5135        .quiet()
5136        .stdin(std::process::Stdio::null())
5137        .output()
5138        .await
5139        .context("spawn gh")?;
5140    if out.status.success() {
5141        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
5142    } else {
5143        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
5144    }
5145}
5146
5147/// Tear a run's worktrees and branches down.
5148pub async fn fold_run(state: &mut RunState, drop_winner: bool) -> Result<Vec<String>> {
5149    let repo = state.repo.clone();
5150    let root = state.worktree_root();
5151    let winner = state.tally.as_ref().map(|t| t.winner);
5152    let mut removed = Vec::new();
5153
5154    for i in 0..state.candidates.len() {
5155        let c = state.candidates[i].clone();
5156        let is_winner = Some(c.label) == winner;
5157        if is_winner && !drop_winner {
5158            continue;
5159        }
5160        if c.worktree.exists() {
5161            git::worktree_remove(&repo, &c.worktree).await.ok();
5162            removed.push(c.worktree.to_string_lossy().into_owned());
5163        }
5164        if git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
5165            git::branch_delete(&repo, &c.branch).await.ok();
5166            removed.push(c.branch.clone());
5167        }
5168        state.candidates[i].folded = true;
5169    }
5170
5171    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
5172        let path = name.path();
5173        let keep = !drop_winner
5174            && winner.is_some_and(|w| {
5175                path.file_name()
5176                    .is_some_and(|n| n == format!("cand-{w}").as_str())
5177            });
5178        if keep {
5179            continue;
5180        }
5181        git::worktree_remove(&repo, &path).await.ok();
5182        removed.push(path.to_string_lossy().into_owned());
5183    }
5184
5185    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
5186    // judge worktrees, so once the loop above has cleared all of them out,
5187    // the parent is a bare directory nobody else was ever going to remove -
5188    // git only ever managed what was inside it. Left alone, one of these
5189    // accumulates per fully-folded run; the operator's own machine had 74.
5190    // `remove_if_empty` re-checks rather than assuming: a run whose winner
5191    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
5192    // so does anything a run never claimed that happens to share the bay.
5193    remove_if_empty(&root);
5194
5195    if state.enabled_worktree_config && drop_winner {
5196        // A release, not a raw disable: some sibling run in this repository
5197        // may still hold its own reference (see `git::acquire_worktree_config`),
5198        // and only the last release actually turns the setting back off.
5199        git::release_worktree_config(&repo).await.ok();
5200        state.enabled_worktree_config = false;
5201    }
5202    state.save()?;
5203    Ok(removed)
5204}
5205
5206/// Remove `dir` if it exists and has nothing in it.
5207///
5208/// Best-effort and silent by design: a directory that is not empty (a run
5209/// whose winner is still parked there, a stray file some other process left)
5210/// is exactly the case this must refuse, and a directory that is already gone
5211/// is not a failure worth reporting either. `std::fs::remove_dir` itself
5212/// already refuses a non-empty directory, so the emptiness check below is
5213/// belt, not suspenders - it is what keeps this from ever attempting the
5214/// removal in the case that matters, rather than trusting `remove_dir`'s
5215/// error path to have no side effects if it ever changed.
5216fn remove_if_empty(dir: &Path) {
5217    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
5218        std::fs::remove_dir(dir).ok();
5219    }
5220}
5221
5222/// Severity of the worst open finding in the last review round, for reporting.
5223pub fn worst_open(state: &RunState) -> Option<Severity> {
5224    state
5225        .reviews
5226        .last()?
5227        .reviews
5228        .iter()
5229        .flat_map(|r| r.findings.iter())
5230        .map(|f| f.severity)
5231        .max()
5232}
5233
5234#[cfg(test)]
5235mod tests {
5236    use super::*;
5237    use crate::run::GateStatus;
5238    use std::collections::BTreeMap;
5239    use std::time::Duration;
5240
5241    fn conductor() -> AgentSpec {
5242        AgentSpec {
5243            id: "conductor".to_owned(),
5244            kind: crate::config::AgentKind::Command,
5245            model: None,
5246            command: vec!["true".to_owned()],
5247            extra_args: Vec::new(),
5248            env: BTreeMap::new(),
5249            prompt_delivery: None,
5250        }
5251    }
5252
5253    #[test]
5254    fn remove_if_empty_only_ever_takes_a_bare_directory() {
5255        let dir = tempfile::tempdir().unwrap();
5256        let bay = dir.path().join("ffff");
5257
5258        // Not there yet: nothing to do, nothing to panic on.
5259        remove_if_empty(&bay);
5260        assert!(!bay.exists());
5261
5262        // Something still inside - the winner's worktree, or a stray file -
5263        // keeps the directory standing.
5264        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
5265        remove_if_empty(&bay);
5266        assert!(bay.exists(), "non-empty directory must survive");
5267
5268        // Once the last entry is gone, so is the directory itself.
5269        std::fs::remove_dir(bay.join("cand-A")).unwrap();
5270        remove_if_empty(&bay);
5271        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
5272    }
5273
5274    // `round_is_clean` is the exact decision this task fixed: a round with a
5275    // seat that never answered must not read the same as a round every seat
5276    // actually reviewed. These are deterministic and process-free by design —
5277    // the equivalent end-to-end check (a real reviewer timing out under a
5278    // live graph run) is a genuine race against wall-clock contention, and a
5279    // spawn slow enough to blow even a generous budget under a loaded test
5280    // run must not turn this specific regression check flaky.
5281
5282    #[test]
5283    fn a_full_panel_that_found_nothing_is_clean() {
5284        assert!(round_is_clean(
5285            0,
5286            true,
5287            2,
5288            2,
5289            0,
5290            IncompleteReviewPolicy::Block
5291        ));
5292    }
5293
5294    #[test]
5295    fn a_missing_seat_is_never_clean_under_the_default_policy() {
5296        assert!(!round_is_clean(
5297            0,
5298            true,
5299            1,
5300            2,
5301            0,
5302            IncompleteReviewPolicy::Block
5303        ));
5304    }
5305
5306    #[test]
5307    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
5308        assert!(!round_is_clean(
5309            1,
5310            true,
5311            1,
5312            2,
5313            0,
5314            IncompleteReviewPolicy::Warn
5315        ));
5316    }
5317
5318    #[test]
5319    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
5320        assert!(round_is_clean(
5321            0,
5322            true,
5323            1,
5324            2,
5325            0,
5326            IncompleteReviewPolicy::Warn
5327        ));
5328    }
5329
5330    #[test]
5331    fn a_full_panel_with_an_open_finding_is_not_clean() {
5332        assert!(!round_is_clean(
5333            1,
5334            true,
5335            2,
5336            2,
5337            0,
5338            IncompleteReviewPolicy::Block
5339        ));
5340    }
5341
5342    #[test]
5343    fn a_full_panel_with_a_red_e2e_is_not_clean() {
5344        assert!(!round_is_clean(
5345            0,
5346            false,
5347            2,
5348            2,
5349            0,
5350            IncompleteReviewPolicy::Block
5351        ));
5352    }
5353
5354    // The stall this task closes: under the default `block` policy, a seat
5355    // missing only because it was rate limited must not force a wait for a
5356    // session limit that will not lift by the next round. `round_is_clean`
5357    // is where that quorum carve-out lives; the review loop around it never
5358    // changes what a reviewer's vote or a finding's severity means.
5359
5360    #[test]
5361    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
5362        // 1 of 2 answered, and the one missing was quota'd — the exact
5363        // "review-2 rate limited (quota)" shape from the field report.
5364        assert!(round_is_clean(
5365            0,
5366            true,
5367            1,
5368            2,
5369            1,
5370            IncompleteReviewPolicy::Block
5371        ));
5372    }
5373
5374    #[test]
5375    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
5376        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
5377        // not a quota loss (`quota_missing` stays 0) — worth another try.
5378        assert!(!round_is_clean(
5379            0,
5380            true,
5381            1,
5382            2,
5383            0,
5384            IncompleteReviewPolicy::Block
5385        ));
5386    }
5387
5388    #[test]
5389    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
5390        assert!(!round_is_clean(
5391            1,
5392            true,
5393            1,
5394            2,
5395            1,
5396            IncompleteReviewPolicy::Block
5397        ));
5398        assert!(!round_is_clean(
5399            0,
5400            false,
5401            1,
5402            2,
5403            1,
5404            IncompleteReviewPolicy::Block
5405        ));
5406    }
5407
5408    #[test]
5409    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
5410        // Every seat quota'd, nobody answered: there is no panel to decide
5411        // on, so this must fall through to the existing block-and-retry
5412        // fallback rather than call an unreviewed patch clean.
5413        assert!(!round_is_clean(
5414            0,
5415            true,
5416            0,
5417            2,
5418            2,
5419            IncompleteReviewPolicy::Block
5420        ));
5421    }
5422
5423    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
5424        CommandOutcome {
5425            command: "test".to_owned(),
5426            code,
5427            output_tail: String::new(),
5428            duration_ms: 0,
5429            resource_blocked,
5430        }
5431    }
5432
5433    #[test]
5434    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
5435        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
5436        assert!(
5437            !verify_inconclusive(&[outcome(Some(1), false)]),
5438            "an ordinary failure is still evidence about the patch"
5439        );
5440        assert!(verify_inconclusive(&[outcome(None, true)]));
5441        assert!(
5442            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
5443            "one inconclusive outcome taints the whole batch"
5444        );
5445        assert!(!verify_inconclusive(&[]));
5446    }
5447
5448    #[tokio::test]
5449    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
5450        // Alive for the first two checks, then dead - confirms the loop
5451        // actually re-polls rather than deciding once and sleeping out the
5452        // ceiling regardless.
5453        let calls = std::sync::atomic::AtomicUsize::new(0);
5454        let started = Instant::now();
5455        wait_for_pids_with(
5456            &[123],
5457            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
5458            Duration::from_millis(5),
5459            Duration::from_secs(5),
5460        )
5461        .await;
5462        assert!(
5463            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
5464            "must keep checking rather than deciding on the first answer"
5465        );
5466        assert!(
5467            started.elapsed() < Duration::from_secs(1),
5468            "must return the moment it is confirmed dead, not wait out the ceiling"
5469        );
5470    }
5471
5472    #[tokio::test]
5473    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
5474        let started = Instant::now();
5475        wait_for_pids_with(
5476            &[123],
5477            |_| true, // never reports dead
5478            Duration::from_millis(5),
5479            Duration::from_millis(30),
5480        )
5481        .await;
5482        let elapsed = started.elapsed();
5483        assert!(
5484            elapsed >= Duration::from_millis(30),
5485            "must not give up before its own ceiling: {elapsed:?}"
5486        );
5487        assert!(
5488            elapsed < Duration::from_secs(1),
5489            "must not wait past its own ceiling either: {elapsed:?}"
5490        );
5491    }
5492
5493    #[tokio::test]
5494    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
5495        let started = Instant::now();
5496        wait_for_pids_with(
5497            &[],
5498            |_| true,
5499            Duration::from_secs(5),
5500            Duration::from_secs(5),
5501        )
5502        .await;
5503        assert!(
5504            started.elapsed() < Duration::from_millis(200),
5505            "an empty pid list has nothing to confirm"
5506        );
5507    }
5508
5509    // `review_conclusion` is the exact decision the review hand-off task
5510    // fixed: a round budget spent (or a tree that stopped moving) must not
5511    // collapse into `Blocked` regardless of what verification actually
5512    // said. Deterministic and process-free for the same reason the
5513    // `round_is_clean` family above is.
5514    fn review_round(
5515        clean: bool,
5516        blocking: usize,
5517        answered: usize,
5518        expected: usize,
5519        progressed: bool,
5520        e2e_ok: bool,
5521    ) -> ReviewRound {
5522        ReviewRound {
5523            round: 1,
5524            head: "h".to_owned(),
5525            verified_head: None,
5526            reviews: Vec::new(),
5527            e2e: vec![CommandOutcome {
5528                command: "test".to_owned(),
5529                code: Some(if e2e_ok { 0 } else { 1 }),
5530                output_tail: String::new(),
5531                duration_ms: 0,
5532                resource_blocked: false,
5533            }],
5534            verify_retried: false,
5535            e2e_deferred: false,
5536            e2e_defer_reason: None,
5537            fix: None,
5538            blocking,
5539            answered,
5540            expected,
5541            clean,
5542            progressed,
5543            vote_split: false,
5544            reconsideration: Vec::new(),
5545            verdict: None,
5546        }
5547    }
5548
5549    #[test]
5550    fn review_conclusion_is_none_when_nothing_has_run() {
5551        assert_eq!(review_conclusion(&[], 3), None);
5552    }
5553
5554    #[test]
5555    fn review_conclusion_is_none_while_rounds_remain() {
5556        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
5557        assert_eq!(review_conclusion(&rounds, 3), None);
5558    }
5559
5560    #[test]
5561    fn review_conclusion_is_gating_once_a_round_is_clean() {
5562        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
5563        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
5564    }
5565
5566    #[test]
5567    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
5568        let rounds = vec![
5569            review_round(false, 1, 2, 2, true, true),
5570            review_round(false, 1, 2, 2, true, true),
5571        ];
5572        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
5573    }
5574
5575    #[test]
5576    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
5577        let rounds = vec![
5578            review_round(false, 1, 2, 2, true, true),
5579            review_round(false, 1, 2, 2, true, false),
5580        ];
5581        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
5582    }
5583
5584    #[test]
5585    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
5586        // Missing input, not a verified tree — never a hand-off candidate.
5587        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
5588        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
5589    }
5590
5591    #[test]
5592    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
5593        let rounds = vec![
5594            review_round(false, 1, 2, 2, false, true),
5595            review_round(false, 1, 2, 2, false, true),
5596        ];
5597        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
5598    }
5599
5600    fn secs(n: u64) -> Duration {
5601        Duration::from_secs(n)
5602    }
5603
5604    /// A throwaway repo with one commit on `main`, for tests that need `merge`
5605    /// to make real (and, if it runs at all, real*ly fail*) git calls.
5606    fn init_repo(dir: &Path) {
5607        let run = |args: &[&str]| {
5608            let out = std::process::Command::new("git")
5609                .args(args)
5610                .current_dir(dir)
5611                .quiet()
5612                .output()
5613                .expect("spawn git");
5614            assert!(
5615                out.status.success(),
5616                "git {args:?} failed: {}",
5617                String::from_utf8_lossy(&out.stderr)
5618            );
5619        };
5620        run(&["init", "-b", "main"]);
5621        run(&["config", "user.name", "magi test"]);
5622        run(&["config", "user.email", "magi@example.com"]);
5623        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
5624        run(&["add", "-A"]);
5625        run(&["commit", "-m", "init"]);
5626    }
5627
5628    // `settle_questions` is what closes the ghost the phone showed: a run's
5629    // seat asked something, the run then ended, and nothing was left to
5630    // abandon the question it left `open`. `HOME` is a process-wide
5631    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
5632    // first time it runs in the binary — every test below still reaches the
5633    // same directory whichever call won, and each gets its own run id from
5634    // `RunState::new`, so they never collide there.
5635    fn ask_test_home() {
5636        crate::run::set_home(std::env::temp_dir().join("magi-graph-ask-tests-home"));
5637    }
5638
5639    /// A minimal, git-free `Runner` at a given status — `settle_questions`
5640    /// reads nothing else off it.
5641    fn runner_at(status: RunStatus) -> Runner {
5642        let mut state = RunState::new(
5643            PathBuf::from("/nonexistent/repo"),
5644            "main".to_owned(),
5645            "deadbeef".to_owned(),
5646            "task".to_owned(),
5647            Config::default(),
5648        );
5649        state.status = status;
5650        Runner {
5651            state,
5652            roles: ResolvedRoles {
5653                implementers: Vec::new(),
5654                judges: Vec::new(),
5655                reviewers: Vec::new(),
5656                fixer: None,
5657                conductor: conductor(),
5658            },
5659            sem: Arc::new(Semaphore::new(1)),
5660            pause: Pause::new(),
5661            interrupt: Pause::new(),
5662        }
5663    }
5664
5665    /// `park_here` folding in the reason `Pause::park_because` recorded -
5666    /// this is what lets an operator reading a run's events tell an
5667    /// interrupt-driven park from an ordinary shutdown park.
5668    #[test]
5669    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
5670        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
5671        let mut runner = runner_at(RunStatus::Implementing);
5672        let interrupt = Pause::new();
5673        runner.watch_interrupt(interrupt.clone());
5674
5675        interrupt.park_because("task a1b2 asked to run first");
5676
5677        assert!(runner.park_here().expect("park_here"));
5678        assert!(runner.state.parked);
5679        let last = runner.state.events.last().expect("a park event");
5680        assert_eq!(last.node, "park");
5681        assert!(
5682            last.message.contains("task a1b2 asked to run first"),
5683            "expected the interrupt reason in {:?}",
5684            last.message
5685        );
5686    }
5687
5688    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
5689    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
5690    /// cleared) must not make a *different* run - one only watching its own,
5691    /// unshared interrupt `Pause` - see itself as parked. If a future change
5692    /// ever collapsed these back into one handle, the interrupt scheduler
5693    /// would park every run for the rest of the daemon's life, not just the
5694    /// one it meant to interrupt.
5695    #[test]
5696    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
5697        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
5698        let mut runner = runner_at(RunStatus::Implementing);
5699        let shutdown = Pause::new();
5700        runner.on_pause(shutdown.clone());
5701        let interrupt = Pause::new();
5702        runner.watch_interrupt(interrupt.clone());
5703
5704        // Nobody has asked for anything yet.
5705        assert!(!runner.park_here().expect("park_here"));
5706        assert!(!runner.state.parked);
5707
5708        // Only the interrupt handle fires; the shutdown handle stays clear.
5709        interrupt.park_because("test");
5710        assert!(!shutdown.parked());
5711        assert!(runner.park_here().expect("park_here"));
5712    }
5713
5714    /// The property every prior attempt at this feature failed to pin down:
5715    /// asking a run to park while one of its nodes has a real, in-flight
5716    /// async operation running (an agent call, in production) must not cut
5717    /// that operation short. `park_here` is only ever consulted *between*
5718    /// `execute`'s node calls - see its own doc - so nothing inside a node
5719    /// can observe a park request until the node itself returns. This proves
5720    /// that structurally, with real `tokio` concurrency and a channel
5721    /// handshake (never a sleep, which would only prove "usually", not
5722    /// "cannot"): the "node" below reports that it has genuinely started,
5723    /// and only then is the park requested; the node still has to be told to
5724    /// finish before `park_here` is ever called, exactly mirroring every
5725    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
5726    /// in `execute`.
5727    #[tokio::test]
5728    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
5729        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
5730        let mut runner = runner_at(RunStatus::Implementing);
5731        let interrupt = Pause::new();
5732        runner.watch_interrupt(interrupt.clone());
5733
5734        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
5735        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
5736
5737        // Stands in for one node's in-flight agent call: it proves it has
5738        // genuinely started, then blocks - exactly as a spawned CLI process
5739        // does - until told to finish.
5740        let node = async move {
5741            started_tx.send(()).expect("send started");
5742            finish_rx.await.expect("recv finish");
5743            "node finished"
5744        };
5745
5746        let interrupter = async move {
5747            started_rx.await.expect("recv started");
5748            // The call is now genuinely in flight. Ask it to park.
5749            interrupt.park_because("higher-priority task waiting");
5750            // Nothing the node does can observe this yet - there is no
5751            // check inside it, by construction - so let the executor run
5752            // anything pending and then let the node finish on its own.
5753            tokio::task::yield_now().await;
5754            finish_tx.send(()).expect("send finish");
5755        };
5756
5757        let (node_result, ()) = tokio::join!(node, interrupter);
5758        assert_eq!(
5759            node_result, "node finished",
5760            "the in-flight call ran to completion"
5761        );
5762
5763        // Only now, at the boundary the real `execute` would check right
5764        // after this node, does the park take effect.
5765        assert!(runner.park_here().expect("park_here"));
5766        assert!(runner.state.parked);
5767    }
5768
5769    /// A run parked mid-competition carries every field it had accumulated
5770    /// through the exact same disk round-trip an ordinary resume uses -
5771    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
5772    /// Nothing about parking for an interrupt is a special case of that path;
5773    /// this is what proves it rather than assuming it.
5774    #[test]
5775    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
5776        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
5777        let mut runner = runner_at(RunStatus::Judging);
5778        // `Runner::resume` re-resolves roles from the saved config, which
5779        // refuses an empty roster - give it the same minimal one `conductor`
5780        // itself uses.
5781        runner.state.config.agents = vec![conductor()];
5782        runner.state.candidates = vec![Candidate {
5783            index: 0,
5784            label: 'A',
5785            agent: "alpha".to_owned(),
5786            branch: "magi/x/A".to_owned(),
5787            worktree: PathBuf::from("/nonexistent/worktree"),
5788            summary: "did the thing".to_owned(),
5789            stat: "1 file changed".to_owned(),
5790            files: 1,
5791            commits: 1,
5792            empty: false,
5793            failed: None,
5794            duration_ms: 1234,
5795            folded: false,
5796        }];
5797        let run_id = runner.state.id.clone();
5798
5799        let interrupt = Pause::new();
5800        runner.watch_interrupt(interrupt.clone());
5801        interrupt.park_because("task c3d4 asked to run first");
5802        assert!(runner.park_here().expect("park_here"));
5803
5804        let resumed = Runner::resume(&run_id).expect("resume");
5805        assert_eq!(resumed.state.candidates.len(), 1);
5806        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
5807        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
5808        assert_eq!(resumed.state.status, runner.state.status);
5809        assert!(
5810            resumed.state.parked,
5811            "still parked until `execute` actually walks the graph again"
5812        );
5813        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
5814    }
5815
5816    /// A fresh open question on `run`, stored and handed back for assertions.
5817    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
5818        let mut q = ask::Question::new(
5819            run.to_owned(),
5820            "implement".to_owned(),
5821            "impl-A".to_owned(),
5822            "Which storage backend should the cache use?".to_owned(),
5823            String::new(),
5824            vec!["SQLite".to_owned(), "Redis".to_owned()],
5825        );
5826        store.put(&mut q).unwrap();
5827        q
5828    }
5829
5830    #[test]
5831    fn a_failed_runs_open_question_is_abandoned() {
5832        ask_test_home();
5833        let store = ask::Questions::open();
5834        let mut runner = runner_at(RunStatus::Failed);
5835        let run = runner.state.id.clone();
5836        let q = ask_open_question(&store, &run);
5837
5838        runner.settle_questions();
5839
5840        let back = store.get(&q.id).unwrap();
5841        assert!(
5842            !back.status.open(),
5843            "the seat that asked died with the run; nobody is left to read an answer"
5844        );
5845        assert!(
5846            back.detail.contains(&run) && back.detail.contains("failed"),
5847            "the reason names what the run became, not just that it is gone: {}",
5848            back.detail
5849        );
5850    }
5851
5852    #[test]
5853    fn a_merged_runs_open_question_is_abandoned_too() {
5854        ask_test_home();
5855        let store = ask::Questions::open();
5856        // A run that finishes cleanly still leaves nobody to read an answer -
5857        // this is not only a failure-path cleanup.
5858        for status in [RunStatus::Merged, RunStatus::Ready] {
5859            let mut runner = runner_at(status);
5860            let run = runner.state.id.clone();
5861            let q = ask_open_question(&store, &run);
5862
5863            runner.settle_questions();
5864
5865            let back = store.get(&q.id).unwrap();
5866            assert!(
5867                !back.status.open(),
5868                "{status:?} run's question must not outlive the run"
5869            );
5870        }
5871    }
5872
5873    #[test]
5874    fn a_still_resumable_runs_open_question_is_left_alone() {
5875        ask_test_home();
5876        let store = ask::Questions::open();
5877        // `Blocked` and `Stalled` can still be resumed — the candidates, the
5878        // review round and the seat sessions are all still on disk — so a
5879        // question asked mid-round may yet get a real answer from a real
5880        // resume. Sweeping it here would be exactly the failure mode this
5881        // whole feature exists to avoid on the other side.
5882        for status in [RunStatus::Blocked, RunStatus::Stalled] {
5883            let mut runner = runner_at(status);
5884            let run = runner.state.id.clone();
5885            let q = ask_open_question(&store, &run);
5886
5887            runner.settle_questions();
5888
5889            let back = store.get(&q.id).unwrap();
5890            assert!(
5891                back.status.open(),
5892                "{status:?} is still alive; the question must still be waiting"
5893            );
5894        }
5895    }
5896
5897    #[test]
5898    fn settle_questions_never_touches_an_already_answered_question() {
5899        ask_test_home();
5900        let store = ask::Questions::open();
5901        let mut runner = runner_at(RunStatus::Failed);
5902        let run = runner.state.id.clone();
5903        let mut q = ask_open_question(&store, &run);
5904        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
5905            .unwrap();
5906        store.put(&mut q).unwrap();
5907
5908        // Called twice, the way a crash-recovered daemon reclaim and the
5909        // graph's own cleanup both can for the same run — `abandon_for_run`
5910        // only ever touches what is still open, so this must be inert both
5911        // times, not merely the second.
5912        runner.settle_questions();
5913        runner.settle_questions();
5914
5915        let back = store.get(&q.id).unwrap();
5916        assert_eq!(
5917            back.status,
5918            ask::QuestionStatus::Answered,
5919            "a real answer is a decision on record, never overwritten by a sweep"
5920        );
5921    }
5922
5923    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
5924    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
5925    /// without merging, whose winner is still the operator's answer to read.
5926    /// Nothing previously called `fold_run` itself with a real `tally`, so
5927    /// this is the first test to pin down the one distinction the whole
5928    /// automatic-fold feature depends on: the winner's worktree and branch
5929    /// must survive, everything else sharing the run's worktree bay - a
5930    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
5931    /// second sweep treats every non-winner directory under the bay alike -
5932    /// must not.
5933    #[tokio::test]
5934    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
5935        crate::run::set_home(std::env::temp_dir().join("magi-graph-fold-run-tests-home"));
5936        let tmp = tempfile::tempdir().expect("tempdir");
5937        let repo = tmp.path().join("repo");
5938        std::fs::create_dir_all(&repo).unwrap();
5939        init_repo(&repo);
5940
5941        let mut config = Config::default();
5942        config.graph.worktree_root = Some(tmp.path().join("wt"));
5943
5944        let mut state = RunState::new(
5945            repo.clone(),
5946            "main".to_owned(),
5947            "deadbeef".to_owned(),
5948            "task".to_owned(),
5949            config,
5950        );
5951        let root = state.worktree_root();
5952        let wt_a = root.join("cand-A");
5953        let wt_b = root.join("cand-B");
5954        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
5955            .await
5956            .expect("worktree A");
5957        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
5958            .await
5959            .expect("worktree B");
5960
5961        state.candidates = vec![
5962            Candidate {
5963                index: 0,
5964                label: 'A',
5965                agent: "alpha".to_owned(),
5966                branch: "magi/x/A".to_owned(),
5967                worktree: wt_a.clone(),
5968                summary: String::new(),
5969                stat: String::new(),
5970                files: 0,
5971                commits: 0,
5972                empty: false,
5973                failed: None,
5974                duration_ms: 0,
5975                folded: false,
5976            },
5977            Candidate {
5978                index: 1,
5979                label: 'B',
5980                agent: "beta".to_owned(),
5981                branch: "magi/x/B".to_owned(),
5982                worktree: wt_b.clone(),
5983                summary: String::new(),
5984                stat: String::new(),
5985                files: 0,
5986                commits: 0,
5987                empty: false,
5988                failed: None,
5989                duration_ms: 0,
5990                folded: false,
5991            },
5992        ];
5993        state.tally = Some(Tally {
5994            first_choice: BTreeMap::from([('A', 1)]),
5995            borda: BTreeMap::new(),
5996            winner: 'A',
5997            rankings: 1,
5998            unanimous_initial: true,
5999            deliberated: false,
6000            changed_votes: 0,
6001            unanimous_final: true,
6002            tie_break: None,
6003            judges: 1,
6004            present: 1,
6005            quorum: 1,
6006            met_quorum: true,
6007            uncontested: None,
6008        });
6009        state.status = RunStatus::Ready;
6010
6011        fold_run(&mut state, false).await.expect("fold_run");
6012
6013        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
6014        assert!(
6015            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
6016            "the unmerged winner's branch survives"
6017        );
6018        assert!(
6019            !state.candidates[0].folded,
6020            "the winner is not marked folded"
6021        );
6022
6023        assert!(!wt_b.exists(), "the loser's worktree is removed");
6024        assert!(
6025            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
6026            "the loser's branch is removed"
6027        );
6028        assert!(state.candidates[1].folded, "the loser is marked folded");
6029    }
6030
6031    /// `status == Ready` used to be read as "this is the harmless
6032    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
6033    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
6034    /// run's PR was closed without merging — and reentering `merge` with
6035    /// `mode` still `Pr` does not know the difference, so it pushed and
6036    /// opened a second pull request. `mode == Local` reproduces the same
6037    /// blind spot without a network call: reentry must not attempt another
6038    /// git merge once this node has already recorded an outcome.
6039    #[tokio::test]
6040    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
6041        let tmp = tempfile::tempdir().expect("tempdir");
6042        let repo = tmp.path().join("repo");
6043        std::fs::create_dir_all(&repo).unwrap();
6044        init_repo(&repo);
6045
6046        let mut config = Config::default();
6047        config.merge.mode = MergeMode::Local;
6048
6049        let mut state = RunState::new(
6050            repo.clone(),
6051            "main".to_owned(),
6052            "deadbeef".to_owned(),
6053            "task".to_owned(),
6054            config,
6055        );
6056        state.candidates = vec![Candidate {
6057            index: 0,
6058            label: 'A',
6059            agent: "alpha".to_owned(),
6060            branch: "does-not-exist".to_owned(),
6061            worktree: repo.clone(),
6062            summary: String::new(),
6063            stat: String::new(),
6064            files: 0,
6065            commits: 0,
6066            empty: false,
6067            failed: None,
6068            duration_ms: 0,
6069            folded: false,
6070        }];
6071        state.tally = Some(Tally {
6072            first_choice: BTreeMap::from([('A', 1)]),
6073            borda: BTreeMap::new(),
6074            winner: 'A',
6075            rankings: 1,
6076            unanimous_initial: true,
6077            deliberated: false,
6078            changed_votes: 0,
6079            unanimous_final: true,
6080            tie_break: None,
6081            judges: 0,
6082            present: 0,
6083            quorum: 0,
6084            met_quorum: true,
6085            uncontested: Some("only candidate A produced a change".to_owned()),
6086        });
6087        state.reviews = vec![ReviewRound {
6088            round: 1,
6089            head: "deadbeef".to_owned(),
6090            verified_head: None,
6091            reviews: Vec::new(),
6092            e2e: Vec::new(),
6093            fix: None,
6094            blocking: 0,
6095            answered: 0,
6096            expected: 0,
6097            clean: true,
6098            verify_retried: false,
6099            e2e_deferred: false,
6100            e2e_defer_reason: None,
6101            progressed: false,
6102            vote_split: false,
6103            reconsideration: Vec::new(),
6104            verdict: None,
6105        }];
6106        state.gate = vec![CommandOutcome {
6107            command: "test".to_owned(),
6108            code: Some(0),
6109            output_tail: String::new(),
6110            duration_ms: 0,
6111            resource_blocked: false,
6112        }];
6113        state.gate_ran = true;
6114        // Reached its conclusion already — e.g. `land` closing the PR without
6115        // merging it, which (like the honest `MergeMode::None` path) leaves
6116        // `status` at `Ready`. The recorded outcome is what actually marks
6117        // this node done.
6118        state.status = RunStatus::Ready;
6119        state.merge = Some(MergeOutcome {
6120            mode: MergeMode::Local,
6121            ok: false,
6122            detail: "already concluded".to_owned(),
6123        });
6124
6125        let mut runner = Runner {
6126            state,
6127            roles: ResolvedRoles {
6128                implementers: Vec::new(),
6129                judges: Vec::new(),
6130                reviewers: Vec::new(),
6131                fixer: None,
6132                conductor: conductor(),
6133            },
6134            sem: Arc::new(Semaphore::new(1)),
6135            pause: Pause::new(),
6136            interrupt: Pause::new(),
6137        };
6138
6139        runner.merge().await.expect("merge");
6140
6141        assert_eq!(
6142            runner.state.status,
6143            RunStatus::Ready,
6144            "a concluded run's status must not change on reentry"
6145        );
6146        assert_eq!(
6147            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
6148            Some("already concluded"),
6149            "merge must not run again once the node already recorded an outcome"
6150        );
6151    }
6152
6153    /// `gate` leaves `state.gate_ran` false both before it has ever run and
6154    /// when its last attempt was resource-blocked (the shared build cache
6155    /// could not be acquired or confirmed fresh in time - see
6156    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
6157    /// `Vec` this also leaves behind used to read as "nothing failed" and let
6158    /// a run merge a tree the gate never actually checked - exactly the case
6159    /// a contended cache produces on every retry until it clears. `merge`
6160    /// must refuse until `gate` has actually recorded an attempt.
6161    #[tokio::test]
6162    async fn merge_refuses_a_gate_that_has_not_actually_run() {
6163        let tmp = tempfile::tempdir().expect("tempdir");
6164        let repo = tmp.path().join("repo");
6165        std::fs::create_dir_all(&repo).unwrap();
6166        init_repo(&repo);
6167
6168        let mut config = Config::default();
6169        config.merge.mode = MergeMode::Local;
6170
6171        let mut state = RunState::new(
6172            repo.clone(),
6173            "main".to_owned(),
6174            "deadbeef".to_owned(),
6175            "task".to_owned(),
6176            config,
6177        );
6178        state.candidates = vec![Candidate {
6179            index: 0,
6180            label: 'A',
6181            agent: "alpha".to_owned(),
6182            branch: "does-not-exist".to_owned(),
6183            worktree: repo.clone(),
6184            summary: String::new(),
6185            stat: String::new(),
6186            files: 0,
6187            commits: 0,
6188            empty: false,
6189            failed: None,
6190            duration_ms: 0,
6191            folded: false,
6192        }];
6193        state.tally = Some(Tally {
6194            first_choice: BTreeMap::from([('A', 1)]),
6195            borda: BTreeMap::new(),
6196            winner: 'A',
6197            rankings: 1,
6198            unanimous_initial: true,
6199            deliberated: false,
6200            changed_votes: 0,
6201            unanimous_final: true,
6202            tie_break: None,
6203            judges: 0,
6204            present: 0,
6205            quorum: 0,
6206            met_quorum: true,
6207            uncontested: Some("only candidate A produced a change".to_owned()),
6208        });
6209        state.reviews = vec![ReviewRound {
6210            round: 1,
6211            head: "deadbeef".to_owned(),
6212            verified_head: None,
6213            reviews: Vec::new(),
6214            e2e: Vec::new(),
6215            fix: None,
6216            blocking: 0,
6217            answered: 0,
6218            expected: 0,
6219            clean: true,
6220            verify_retried: false,
6221            e2e_deferred: false,
6222            e2e_defer_reason: None,
6223            progressed: false,
6224            vote_split: false,
6225            reconsideration: Vec::new(),
6226            verdict: None,
6227        }];
6228        // The point: `gate` has not recorded anything yet.
6229        state.gate = Vec::new();
6230        state.gate_ran = false;
6231        state.status = RunStatus::Gating;
6232
6233        let mut runner = Runner {
6234            state,
6235            roles: ResolvedRoles {
6236                implementers: Vec::new(),
6237                judges: Vec::new(),
6238                reviewers: Vec::new(),
6239                fixer: None,
6240                conductor: conductor(),
6241            },
6242            sem: Arc::new(Semaphore::new(1)),
6243            pause: Pause::new(),
6244            interrupt: Pause::new(),
6245        };
6246
6247        runner.merge().await.expect("merge");
6248
6249        assert!(
6250            runner.state.merge.is_none(),
6251            "an empty gate must never be read as a passing one: {:?}",
6252            runner.state.merge
6253        );
6254    }
6255
6256    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
6257    /// commands configured and `merge.mode` is `none` (a review-only run).
6258    /// `gate` must still record a real attempt — zero commands, vacuously
6259    /// passed — rather than leaving `state.gate` empty in a way `merge`
6260    /// cannot tell apart from "never ran"; otherwise the run reaches
6261    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
6262    #[tokio::test]
6263    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
6264        let tmp = tempfile::tempdir().expect("tempdir");
6265        let repo = tmp.path().join("repo");
6266        std::fs::create_dir_all(&repo).unwrap();
6267        init_repo(&repo);
6268
6269        // Default config: `verify.gate` empty, `merge.mode` is `none`.
6270        let config = Config::default();
6271
6272        let mut state = RunState::new(
6273            repo.clone(),
6274            "main".to_owned(),
6275            "deadbeef".to_owned(),
6276            "task".to_owned(),
6277            config,
6278        );
6279        state.candidates = vec![Candidate {
6280            index: 0,
6281            label: 'A',
6282            agent: "alpha".to_owned(),
6283            branch: "does-not-exist".to_owned(),
6284            worktree: repo.clone(),
6285            summary: String::new(),
6286            stat: String::new(),
6287            files: 0,
6288            commits: 0,
6289            empty: false,
6290            failed: None,
6291            duration_ms: 0,
6292            folded: false,
6293        }];
6294        state.tally = Some(Tally {
6295            first_choice: BTreeMap::from([('A', 1)]),
6296            borda: BTreeMap::new(),
6297            winner: 'A',
6298            rankings: 1,
6299            unanimous_initial: true,
6300            deliberated: false,
6301            changed_votes: 0,
6302            unanimous_final: true,
6303            tie_break: None,
6304            judges: 0,
6305            present: 0,
6306            quorum: 0,
6307            met_quorum: true,
6308            uncontested: Some("only candidate A produced a change".to_owned()),
6309        });
6310        state.reviews = vec![ReviewRound {
6311            round: 1,
6312            head: "deadbeef".to_owned(),
6313            verified_head: None,
6314            reviews: Vec::new(),
6315            e2e: Vec::new(),
6316            fix: None,
6317            blocking: 0,
6318            answered: 0,
6319            expected: 0,
6320            clean: true,
6321            verify_retried: false,
6322            e2e_deferred: false,
6323            e2e_defer_reason: None,
6324            progressed: false,
6325            vote_split: false,
6326            reconsideration: Vec::new(),
6327            verdict: None,
6328        }];
6329
6330        let mut runner = Runner {
6331            state,
6332            roles: ResolvedRoles {
6333                implementers: Vec::new(),
6334                judges: Vec::new(),
6335                reviewers: Vec::new(),
6336                fixer: None,
6337                conductor: conductor(),
6338            },
6339            sem: Arc::new(Semaphore::new(1)),
6340            pause: Pause::new(),
6341            interrupt: Pause::new(),
6342        };
6343
6344        runner.gate().await.expect("gate");
6345        assert!(
6346            runner.state.gate_ran,
6347            "zero configured commands is still a real attempt, not an unrun gate"
6348        );
6349        assert!(runner.state.gate.is_empty());
6350        assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
6351        assert_ne!(
6352            runner.state.status,
6353            RunStatus::Blocked,
6354            "a gate with nothing to check must not read as failed"
6355        );
6356
6357        runner.merge().await.expect("merge");
6358        assert_eq!(
6359            runner.state.status,
6360            RunStatus::Ready,
6361            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
6362        );
6363    }
6364
6365    /// `Config::cache_dir` is derived from `verify.e2e` as well as
6366    /// `verify.gate` (so the e2e leg and the final gate never build against
6367    /// different directories). With zero `verify.gate` commands but a
6368    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
6369    /// that lease before discovering it had nothing to run - so a repo with
6370    /// no gate commands could come back `resource_blocked` (and therefore
6371    /// still `gate_ran == false`) on nothing but an unrelated run holding the
6372    /// cache, exactly the contention this run's own zero commands could
6373    /// never have touched. `gate` must recognise there is nothing to check
6374    /// before it ever asks for the lease.
6375    #[tokio::test]
6376    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
6377        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
6378        let home = crate::run::home();
6379
6380        let tmp = tempfile::tempdir().expect("tempdir");
6381        let repo = tmp.path().join("repo");
6382        std::fs::create_dir_all(&repo).unwrap();
6383        init_repo(&repo);
6384        // Unique to this test, so holding its lease cannot collide with
6385        // another test sharing the same process-wide `home`.
6386        let cache_dir = tmp.path().join("target");
6387
6388        let mut config = Config::default();
6389        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
6390        // `verify.gate` stays empty (the default). Bounded so a regression
6391        // that does start waiting fails the test in seconds, not hangs it.
6392        config.graph.timeout_verify = Some(2);
6393
6394        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
6395        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
6396            .expect("no io error acquiring directly")
6397        {
6398            crate::cache::AcquireOutcome::Acquired(g) => g,
6399            crate::cache::AcquireOutcome::Busy(b) => {
6400                panic!("expected the direct acquire to win the lease first: {b:?}")
6401            }
6402        };
6403
6404        let mut state = RunState::new(
6405            repo.clone(),
6406            "main".to_owned(),
6407            "deadbeef".to_owned(),
6408            "task".to_owned(),
6409            config,
6410        );
6411        state.candidates = vec![Candidate {
6412            index: 0,
6413            label: 'A',
6414            agent: "alpha".to_owned(),
6415            branch: "does-not-exist".to_owned(),
6416            worktree: repo.clone(),
6417            summary: String::new(),
6418            stat: String::new(),
6419            files: 0,
6420            commits: 0,
6421            empty: false,
6422            failed: None,
6423            duration_ms: 0,
6424            folded: false,
6425        }];
6426        state.tally = Some(Tally {
6427            first_choice: BTreeMap::from([('A', 1)]),
6428            borda: BTreeMap::new(),
6429            winner: 'A',
6430            rankings: 1,
6431            unanimous_initial: true,
6432            deliberated: false,
6433            changed_votes: 0,
6434            unanimous_final: true,
6435            tie_break: None,
6436            judges: 0,
6437            present: 0,
6438            quorum: 0,
6439            met_quorum: true,
6440            uncontested: Some("only candidate A produced a change".to_owned()),
6441        });
6442        state.reviews = vec![ReviewRound {
6443            round: 1,
6444            head: "deadbeef".to_owned(),
6445            verified_head: None,
6446            reviews: Vec::new(),
6447            e2e: Vec::new(),
6448            fix: None,
6449            blocking: 0,
6450            answered: 0,
6451            expected: 0,
6452            clean: true,
6453            verify_retried: false,
6454            e2e_deferred: false,
6455            e2e_defer_reason: None,
6456            progressed: false,
6457            vote_split: false,
6458            reconsideration: Vec::new(),
6459            verdict: None,
6460        }];
6461
6462        let mut runner = Runner {
6463            state,
6464            roles: ResolvedRoles {
6465                implementers: Vec::new(),
6466                judges: Vec::new(),
6467                reviewers: Vec::new(),
6468                fixer: None,
6469                conductor: conductor(),
6470            },
6471            sem: Arc::new(Semaphore::new(1)),
6472            pause: Pause::new(),
6473            interrupt: Pause::new(),
6474        };
6475
6476        let started = std::time::Instant::now();
6477        runner.gate().await.expect("gate");
6478        assert!(
6479            started.elapsed() < Duration::from_secs(1),
6480            "a gate with nothing to run must never wait on a lease it never needed"
6481        );
6482        assert!(
6483            runner.state.gate_ran,
6484            "zero commands is still a real, immediate attempt"
6485        );
6486        assert!(runner.state.gate.is_empty());
6487        assert_ne!(
6488            runner.state.status,
6489            RunStatus::Blocked,
6490            "must not read as resource-blocked on a lease it never asked for"
6491        );
6492    }
6493
6494    #[tokio::test]
6495    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
6496        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
6497        let tmp = tempfile::tempdir().expect("tempdir");
6498        let repo = tmp.path().join("repo");
6499        std::fs::create_dir_all(&repo).unwrap();
6500        init_repo(&repo);
6501
6502        let mut config = Config::default();
6503        config.merge.mode = MergeMode::Pr;
6504        config.graph.land = true;
6505        config.graph.land_approval = false;
6506
6507        let mut state = RunState::new(
6508            repo.clone(),
6509            "main".to_owned(),
6510            "deadbeef".to_owned(),
6511            "task".to_owned(),
6512            config,
6513        );
6514        state.candidates = vec![Candidate {
6515            index: 0,
6516            label: 'A',
6517            agent: "alpha".to_owned(),
6518            branch: "does-not-exist".to_owned(),
6519            worktree: repo.clone(),
6520            summary: String::new(),
6521            stat: String::new(),
6522            files: 0,
6523            commits: 0,
6524            empty: false,
6525            failed: None,
6526            duration_ms: 0,
6527            folded: false,
6528        }];
6529        state.tally = Some(Tally {
6530            first_choice: BTreeMap::from([('A', 1)]),
6531            borda: BTreeMap::new(),
6532            winner: 'A',
6533            rankings: 1,
6534            unanimous_initial: true,
6535            deliberated: false,
6536            changed_votes: 0,
6537            unanimous_final: true,
6538            tie_break: None,
6539            judges: 0,
6540            present: 0,
6541            quorum: 0,
6542            met_quorum: true,
6543            uncontested: Some("only candidate A produced a change".to_owned()),
6544        });
6545        state.reviews = vec![ReviewRound {
6546            round: 1,
6547            head: "deadbeef".to_owned(),
6548            verified_head: None,
6549            reviews: Vec::new(),
6550            e2e: Vec::new(),
6551            fix: None,
6552            blocking: 0,
6553            answered: 0,
6554            expected: 0,
6555            clean: true,
6556            verify_retried: false,
6557            e2e_deferred: false,
6558            e2e_defer_reason: None,
6559            progressed: false,
6560            vote_split: false,
6561            reconsideration: Vec::new(),
6562            verdict: None,
6563        }];
6564        state.gate = vec![CommandOutcome {
6565            command: "test".to_owned(),
6566            code: Some(0),
6567            output_tail: String::new(),
6568            duration_ms: 0,
6569            resource_blocked: false,
6570        }];
6571        state.gate_ran = true;
6572        // A first pass through `merge` already pushed and opened this pull
6573        // request; `status` is `Landing` because a previous call into `land`
6574        // parked or was interrupted before it reached a terminal outcome.
6575        state.status = RunStatus::Landing;
6576        state.merge = Some(MergeOutcome {
6577            mode: MergeMode::Pr,
6578            ok: true,
6579            detail: "https://example.invalid/x/y/pull/1".to_owned(),
6580        });
6581
6582        // The Landing-resume shortcut calls `run_land` directly rather than
6583        // through `merge`, which is exactly the call site that used to skip
6584        // `settle_questions` - see the fixture below.
6585        ask_test_home();
6586        let store = ask::Questions::open();
6587        let q = ask_open_question(&store, &state.id);
6588
6589        let mut runner = Runner {
6590            state,
6591            roles: ResolvedRoles {
6592                implementers: Vec::new(),
6593                judges: Vec::new(),
6594                reviewers: Vec::new(),
6595                fixer: None,
6596                conductor: conductor(),
6597            },
6598            sem: Arc::new(Semaphore::new(1)),
6599            pause: Pause::new(),
6600            interrupt: Pause::new(),
6601        };
6602
6603        // `execute`, not `merge` directly: the Landing-resume shortcut lives
6604        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
6605        // exactly because `review_loop` would otherwise clobber the marker
6606        // first.
6607        runner.execute().await.expect("execute");
6608
6609        assert_eq!(
6610            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
6611            Some("https://example.invalid/x/y/pull/1"),
6612            "reentry must not push again or open a second pull request over the \
6613             one `land` is already watching"
6614        );
6615        assert_ne!(
6616            runner.state.status,
6617            RunStatus::Landing,
6618            "land could not actually reach the fake pull request, so it must \
6619             have given up rather than left the run silently parked forever"
6620        );
6621        // `land` could not reach the fake pull request, so it gave up into
6622        // `Blocked` - still resumable, so the question must not have been
6623        // swept just because this branch now also calls `settle_questions`.
6624        assert_eq!(runner.state.status, RunStatus::Blocked);
6625        assert!(
6626            store.get(&q.id).unwrap().status.open(),
6627            "Blocked is still alive; settle_questions must have been a no-op here"
6628        );
6629    }
6630
6631    fn state_with_round(round: ReviewRound) -> RunState {
6632        let mut s = RunState::new(
6633            PathBuf::from("/repo"),
6634            "main".to_owned(),
6635            "abc1234".to_owned(),
6636            "add retries".to_owned(),
6637            Config::default(),
6638        );
6639        s.reviews = vec![round];
6640        s
6641    }
6642
6643    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
6644        crate::verdict::Finding {
6645            id: id.to_owned(),
6646            severity,
6647            file: None,
6648            line: None,
6649            title: title.to_owned(),
6650            detail: String::new(),
6651        }
6652    }
6653
6654    #[test]
6655    fn pr_body_names_open_findings_and_declined_ones() {
6656        let round = ReviewRound {
6657            round: 2,
6658            head: "deadbee".to_owned(),
6659            verified_head: None,
6660            reviews: vec![ReviewRecord {
6661                reviewer: 1,
6662                agent: "alpha".to_owned(),
6663                summary: String::new(),
6664                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
6665                vote: None,
6666                failed: None,
6667                duration_ms: 0,
6668            }],
6669            e2e: vec![CommandOutcome {
6670                command: "cargo test".to_owned(),
6671                code: Some(0),
6672                output_tail: String::new(),
6673                duration_ms: 0,
6674                resource_blocked: false,
6675            }],
6676            verify_retried: false,
6677            e2e_deferred: false,
6678            e2e_defer_reason: None,
6679            fix: Some(FixRecord {
6680                agent: "alpha".to_owned(),
6681                addressed: Vec::new(),
6682                rejected: vec![crate::verdict::Rejection {
6683                    id: "R1-1-1".to_owned(),
6684                    why: "not reachable from any caller".to_owned(),
6685                }],
6686                notes: String::new(),
6687                committed: true,
6688                failed: None,
6689                duration_ms: 0,
6690                continuation: None,
6691            }),
6692            blocking: 0,
6693            answered: 1,
6694            expected: 1,
6695            clean: false,
6696            progressed: true,
6697            vote_split: false,
6698            reconsideration: Vec::new(),
6699            verdict: None,
6700        };
6701        let state = state_with_round(round);
6702        let body = pr_body(&state, 'A');
6703
6704        assert!(body.contains("add retries"), "the task must still be there");
6705        assert!(body.contains("R2-1-1"), "{body}");
6706        assert!(body.contains("unused import"), "{body}");
6707        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
6708        assert!(
6709            body.contains("not reachable from any caller"),
6710            "the reason it was declined: {body}"
6711        );
6712    }
6713
6714    #[test]
6715    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
6716        let round = ReviewRound {
6717            round: 1,
6718            head: "deadbee".to_owned(),
6719            verified_head: None,
6720            reviews: vec![ReviewRecord {
6721                reviewer: 1,
6722                agent: "alpha".to_owned(),
6723                summary: String::new(),
6724                findings: Vec::new(),
6725                vote: None,
6726                failed: None,
6727                duration_ms: 0,
6728            }],
6729            e2e: Vec::new(),
6730            verify_retried: false,
6731            e2e_deferred: false,
6732            e2e_defer_reason: None,
6733            fix: None,
6734            blocking: 0,
6735            answered: 1,
6736            expected: 1,
6737            clean: true,
6738            progressed: false,
6739            vote_split: false,
6740            reconsideration: Vec::new(),
6741            verdict: None,
6742        };
6743        let state = state_with_round(round);
6744        let body = pr_body(&state, 'A');
6745        assert!(!body.contains("Open review findings"), "{body}");
6746        assert!(!body.contains("Declined"), "{body}");
6747    }
6748
6749    #[test]
6750    fn pr_body_titles_itself_from_the_task_not_run_or_candidate() {
6751        let state = RunState::new(
6752            PathBuf::from("/repo"),
6753            "main".to_owned(),
6754            "abc1234".to_owned(),
6755            "add retries".to_owned(),
6756            Config::default(),
6757        );
6758        let body = pr_body(&state, 'A');
6759        let title = body.lines().next().unwrap();
6760
6761        assert_eq!(
6762            title, "add retries",
6763            "the title must be the task, not run/candidate bookkeeping: {body}"
6764        );
6765        assert!(
6766            body.contains(&format!("magi:run/{}", state.id)),
6767            "the run id must still be recoverable from the footer: {body}"
6768        );
6769        assert!(
6770            body.contains("magi:candidate-a"),
6771            "the candidate must still be recoverable from the footer: {body}"
6772        );
6773    }
6774
6775    #[test]
6776    fn pr_body_never_titles_itself_off_a_blank_first_line() {
6777        let leading_blank = RunState::new(
6778            PathBuf::from("/repo"),
6779            "main".to_owned(),
6780            "abc1234".to_owned(),
6781            "\n\n  \nadd retries\n\ndetails".to_owned(),
6782            Config::default(),
6783        );
6784        let body = pr_body(&leading_blank, 'A');
6785        assert_eq!(
6786            body.lines().next(),
6787            Some("add retries"),
6788            "a leading blank line must not become an empty title: {body}"
6789        );
6790
6791        let whitespace_only = RunState::new(
6792            PathBuf::from("/repo"),
6793            "main".to_owned(),
6794            "abc1234".to_owned(),
6795            "   \n  \n".to_owned(),
6796            Config::default(),
6797        );
6798        let body = pr_body(&whitespace_only, 'A');
6799        let title = body.lines().next().unwrap_or_default();
6800        assert!(
6801            !title.is_empty(),
6802            "a whitespace-only instruction must still fall back to a non-empty title: {body}"
6803        );
6804    }
6805
6806    #[test]
6807    fn pr_title_truncates_a_first_line_over_githubs_limit() {
6808        // A run 2963-shaped instruction: a single first line well past
6809        // GitHub's 256-character createPullRequest limit, with a multi-byte
6810        // character mixed in so the truncation is exercised on `chars()`
6811        // counting rather than bytes.
6812        let long_line = format!("fix the thing 🎉 {}", "x".repeat(400));
6813        let title = pr_title(&long_line);
6814
6815        assert!(
6816            title.chars().count() <= PR_TITLE_MAX,
6817            "title must stay within PR_TITLE_MAX: {title:?} ({} chars)",
6818            title.chars().count()
6819        );
6820        assert!(
6821            title.chars().count() < 256,
6822            "title must stay within GitHub's 256-character limit: {title:?}"
6823        );
6824        assert!(
6825            title.ends_with('…'),
6826            "a truncated title must say so: {title:?}"
6827        );
6828    }
6829
6830    #[test]
6831    fn pr_title_leaves_a_short_title_untouched() {
6832        let title = pr_title("add retries\n\nmore detail below");
6833        assert_eq!(title, "add retries");
6834    }
6835
6836    #[test]
6837    fn pr_title_strips_markdown_heading_markers() {
6838        let title = pr_title("# Rework the config loader\n\ndetails");
6839        assert_eq!(title, "Rework the config loader");
6840    }
6841
6842    #[test]
6843    fn pr_title_of_pr_body_stays_within_githubs_limit() {
6844        let state = RunState::new(
6845            PathBuf::from("/repo"),
6846            "main".to_owned(),
6847            "abc1234".to_owned(),
6848            format!("fix the thing 🎉 {}", "x".repeat(400)),
6849            Config::default(),
6850        );
6851        let body = pr_body(&state, 'A');
6852        let title = pr_title(&body);
6853
6854        assert!(
6855            title.chars().count() < 256,
6856            "the title gh_pr_create sends must stay within GitHub's limit: {title:?}"
6857        );
6858    }
6859
6860    #[test]
6861    fn manual_merge_command_matches_the_configured_style() {
6862        let repo = Path::new("/repo");
6863        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
6864
6865        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
6866        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
6867
6868        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
6869        assert_eq!(
6870            squash,
6871            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
6872             \"Merge magi run 0832 (candidate A)\""
6873        );
6874
6875        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
6876        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
6877    }
6878
6879    #[test]
6880    fn a_nudge_gets_a_quarter_of_the_budget() {
6881        // The judge and implement budgets magi ships with.
6882        assert_eq!(retry_budget(secs(1200), true), secs(300));
6883        assert_eq!(retry_budget(secs(3600), true), secs(900));
6884    }
6885
6886    #[test]
6887    fn a_resent_prompt_keeps_the_whole_budget() {
6888        // The seat kept no context, so the retry is the original job again and
6889        // shortening it would only guarantee a second failure.
6890        assert_eq!(retry_budget(secs(1200), false), secs(1200));
6891        assert_eq!(retry_budget(secs(60), false), secs(60));
6892    }
6893
6894    #[test]
6895    fn the_floor_never_exceeds_the_original_budget() {
6896        // A short configured timeout must not be *raised* by the floor: the
6897        // operator asked for a bound, and a retry may not outlast the attempt
6898        // it is retrying.
6899        assert_eq!(retry_budget(secs(60), true), secs(60));
6900        assert_eq!(retry_budget(secs(480), true), secs(120));
6901        assert_eq!(retry_budget(secs(0), true), secs(0));
6902    }
6903
6904    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
6905        agent::CommandEvidence {
6906            id: "item1".to_owned(),
6907            description: "cargo test".to_owned(),
6908            exit_code,
6909            result_summary: String::new(),
6910            source: "codex".to_owned(),
6911        }
6912    }
6913
6914    #[test]
6915    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
6916        // No evidence is not the same fact as unconfirmed evidence: a
6917        // backend with no adapter, or a reply that ran no commands at all,
6918        // must not be misread as carrying a dangling job.
6919        assert!(!has_unconfirmed_command(&[]));
6920    }
6921
6922    #[test]
6923    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
6924        // Deliberately not a check on the exit code's *value*: a fixer
6925        // legitimately runs something that fails mid-iteration before it
6926        // succeeds, and that must never by itself reopen a valid report.
6927        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
6928        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
6929        assert!(!has_unconfirmed_command(&[
6930            evidence(Some(0)),
6931            evidence(Some(101))
6932        ]));
6933    }
6934
6935    #[test]
6936    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
6937        assert!(has_unconfirmed_command(&[
6938            evidence(Some(0)),
6939            evidence(None)
6940        ]));
6941    }
6942}