Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::git;
39use crate::land;
40use crate::proc::Quiet as _;
41use crate::prompt::{
42    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
43};
44use crate::queue;
45use crate::run::{
46    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
47    DeliberationRound, DeliberationTurn, E2eStatus, FixRecord, JobRecord, JobStatus, Judgement,
48    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, QuotaLoss,
49    ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, Tally, VoteRecord, tail,
50    write_artifact,
51};
52use crate::verdict::{
53    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
54    ReviewVote, Severity,
55};
56
57/// How much verification output is kept and fed back to the fixer.
58const OUTPUT_TAIL: usize = 8_000;
59
60/// Bytes of a failing command's output kept in an event, so the reason a run
61/// stopped is readable from the report without opening `run.json`.
62const EVENT_OUTPUT_TAIL: usize = 2_000;
63
64/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
65/// command's pid before releasing the build cache's lease.
66const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
67
68/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
69/// command's pid to actually exit before giving up and releasing anyway.
70///
71/// A timeout means the process was asked to die (`kill_on_drop`,
72/// `start_kill`), not that it already has — on Windows in particular that can
73/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
74/// the instant the command returns would let the very next acquirer (this
75/// run's own next round, another run's verification, the janitor's prune)
76/// start touching the same directory while it might still be writing to it,
77/// so this polls the actual pid — real confirmation, not a fixed guess —
78/// until it is gone or this ceiling is reached. It is still not full
79/// process-tree reaping: a grandchild the timed-out process spawned and that
80/// outlives it independently is invisible to a pid check, and continuing to
81/// observe and collect *that* stays a different piece of work with its own
82/// owner. Set generously because the common case returns early the moment
83/// the pid is confirmed gone, not because every timeout pays this in full.
84const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
85
86/// Consecutive review rounds with no tree progress (see
87/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
88/// instead of spending the rest of the round budget.
89///
90/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
91/// legitimately finds nothing left to change (its previous round's fix already
92/// covered it, and this round's reviewers re-raised only nits) looks the same
93/// as one that is spinning, for exactly one round. Two in a row is where the
94/// two stop being distinguishable, and a review round on this workload has
95/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
96/// third attempt at a tree that has not moved twice running is pure cost.
97/// This does not touch `review_rounds` itself, which stays the operator's
98/// call.
99pub(crate) const STAGNANT_LIMIT: usize = 2;
100
101/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
102/// a base that moved before giving up and leaving the run `Blocked` for a
103/// person.
104///
105/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
106/// that keeps moving faster than a run can catch it is not something more
107/// rebasing fixes, it is a person's call. Not the same *number as*
108/// `land_rounds` - this budget is spent before a pull request exists, land's
109/// after - but bounded for the identical reason, so it uses the same
110/// default. Counted across both call sites in [`Runner::finish_after_tally`]
111/// (once before review, once before the gate), because either one finding
112/// the base still moving is the same signal.
113const BASE_SYNC_ROUNDS: usize = 4;
114
115/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
116/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
117/// reply held no [`FixReport`].
118///
119/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
120/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
121/// "I'll pause here until the `cargo make check` background run reports
122/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
123/// No `FixReport` was ever collected from that seat, and the run moved on to
124/// the next review round regardless.
125///
126/// Bounded independently of `review_rounds` and `graph.retries`: this
127/// recovers one seat's missing report mid-round, not a new round of review or
128/// an ordinary parse retry, and must not itself become the unbounded wait the
129/// rest of this module exists to avoid.
130const MAX_FIX_CONTINUATIONS: usize = 2;
131
132/// One queued agent invocation.
133///
134/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
135/// CLI hung up on its own stream is asked again from the same job rather than
136/// rebuilt from scratch. See [`Runner::resume_undelivered`].
137#[derive(Clone)]
138struct SeatJob {
139    spec: AgentSpec,
140    seat: SeatState,
141    cwd: PathBuf,
142    prompt: String,
143    timeout: Duration,
144    allow_write: bool,
145    sessions: bool,
146    artifacts: PathBuf,
147    stem: String,
148}
149
150/// How the graph reads one agent invocation.
151///
152/// Quota is split out from an ordinary failure on purpose: a rate-limited call
153/// is known to fail again if retried now, so the retry loop must not spend an
154/// attempt on it. `Dropped` is split out for the opposite reason: unlike
155/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
156/// error JSON, never the agent's answer — a caller that matched only
157/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
158/// left to read that JSON as if it were usable output. `resume_undelivered`
159/// is the only caller that acts on it; everywhere else it is reported like an
160/// ordinary failure.
161enum AgentOutcome {
162    /// A usable output.
163    Ok(AgentOutput),
164    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
165    Quota(AgentOutput),
166    /// The CLI hung up on its own stream after billed work. See
167    /// [`agent::AgentOutput::work_undelivered`].
168    Dropped(AgentOutput),
169    /// Any other failure: a timeout, a bad exit code, an empty reply.
170    Failed(String),
171}
172
173/// A request to park the run at its next node boundary.
174///
175/// Cloning is how the request travels: the loop keeps one handle and hands a
176/// clone to each [`Runner`], and every clone points at the same flag. There
177/// is no channel because there is nothing to send - the only message is
178/// "park", it is idempotent, and a flag cannot be missed by a receiver that
179/// was not listening yet.
180///
181/// The boundary is what makes this cheap. Every node writes the run's state
182/// before the next one starts, and every node skips what is already recorded:
183/// `prep` returns early once candidates exist, `implement` asks only the seats
184/// with nothing on disk, `judge` returns early once judgements exist. So a
185/// parked run resumes into exactly the node it stopped before, and no agent
186/// work is thrown away. Killing the process mid-node, by contrast, loses
187/// whatever the seats in flight had not yet written - which for an implement
188/// wave is an hour of paid work.
189///
190/// A [`Runner`] watches two independent handles of this type - see
191/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
192/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
193/// clone covering the whole daemon's lifetime and is never asked to un-park,
194/// which is correct exactly because nothing is dispatched after it fires.
195/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
196/// that parks for an interrupted task must go on to run other tasks
197/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
198/// reusing the daemon-wide one.
199#[derive(Debug, Clone, Default)]
200pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
201
202impl Pause {
203    /// A pause nobody has asked for yet.
204    #[must_use]
205    pub fn new() -> Self {
206        Self::default()
207    }
208
209    /// Ask the run to park at its next node boundary. Idempotent.
210    pub fn park(&self) {
211        self.0.store(true, Ordering::SeqCst);
212    }
213
214    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
215    /// fold into the run's own `park` event - so an operator reading the run
216    /// later knows this was a deliberate interrupt rather than a shutdown or
217    /// a binary swap. The first reason recorded wins; a park already in
218    /// flight is not relabelled by a second, unrelated request.
219    pub fn park_because(&self, reason: impl Into<String>) {
220        let mut reason_guard = self
221            .1
222            .lock()
223            .unwrap_or_else(std::sync::PoisonError::into_inner);
224        if reason_guard.is_none() {
225            *reason_guard = Some(reason.into());
226        }
227        drop(reason_guard);
228        self.park();
229    }
230
231    /// Has a park been asked for?
232    #[must_use]
233    pub fn parked(&self) -> bool {
234        self.0.load(Ordering::SeqCst)
235    }
236
237    /// Why the park was asked for, when the caller used [`Pause::park_because`].
238    #[must_use]
239    pub fn reason(&self) -> Option<String> {
240        self.1
241            .lock()
242            .unwrap_or_else(std::sync::PoisonError::into_inner)
243            .clone()
244    }
245}
246
247/// Drives one run.
248pub struct Runner {
249    /// Run state; public so the CLI can report on it.
250    pub state: RunState,
251    roles: ResolvedRoles,
252    sem: Arc<Semaphore>,
253    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
254    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
255    /// this is never the same handle as `interrupt`.
256    pause: Pause,
257    /// Set when `magi serve`'s interrupt scheduler wants this specific run
258    /// parked at its next node boundary, to let a task marked
259    /// [`crate::queue::Task::interrupt`] run alone before this one carries
260    /// on. Unlike `pause`, a fresh, unshared handle per run - see
261    /// [`Runner::watch_interrupt`].
262    interrupt: Pause,
263}
264
265/// The commit a run branches from: the base branch as the remote has it.
266///
267/// Two failures this replaces. A run used to branch off `HEAD` and so refused
268/// to start on a dirty tree, which made `magi serve` decline every task for as
269/// long as the operator had work in progress - most of the time. Branching off
270/// the *local* base branch fixed that and introduced a worse one: `land` merges
271/// the winner on GitHub, nothing updates the local ref, and the next run
272/// branches off a base missing everything the previous runs landed. Two tasks
273/// in a row from a phone would have had the second silently re-implementing
274/// against stale code and opening a pull request that reverted the first.
275///
276/// Only refs move here - no checkout, no local branch, no merge - so it is safe
277/// with uncommitted work in the tree. A machine with no network still starts:
278/// the fetch may fail and the local tip is used with a warning, because
279/// refusing to run offline is a worse failure than running against a base the
280/// operator can see for themselves.
281///
282/// One function, called by both entry points. Two answers to "where does a run
283/// branch from" is the kind of drift nobody notices until a diff is wrong.
284async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
285    let tracking = format!("{remote}/{base_branch}");
286    let fetched = git::fetch(repo, remote, base_branch).await;
287    if let Ok(out) = &fetched
288        && out.ok()
289        && git::rev_exists(repo, &tracking).await
290    {
291        return git::rev_parse(repo, &tracking).await;
292    }
293    let why = match &fetched {
294        Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
295        Ok(_) => format!("{remote} has no {base_branch}"),
296        Err(e) => e.to_string(),
297    };
298    tracing::warn!(
299        "could not read {tracking} ({why}); branching off the local \
300         {base_branch} instead, which may be behind"
301    );
302    git::rev_parse(repo, base_branch).await.with_context(|| {
303        format!(
304            "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
305             branch that exists"
306        )
307    })
308}
309
310/// Exclusive claim on one run's `magi fix` step, released on drop — including
311/// on an early return or a panic.
312///
313/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
314/// manual `magi fix` invocations against the same run are otherwise
315/// invisible to each other and would race to remove and recreate the same
316/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
317/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
318/// which is only ever reclaimed later, out of band, by
319/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
320/// `magi fix` invocation is not necessarily running under either of those, so
321/// nothing would ever sweep a lock a killed or crashed process left behind.
322/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
323/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
324/// lease use: an unreadable or unparsable pid, or a liveness query the
325/// platform cannot answer, reads as alive and the lock is left in place.
326struct FixClaim {
327    path: PathBuf,
328}
329
330impl FixClaim {
331    fn acquire(dir: &Path) -> Result<Self> {
332        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
333        let path = dir.join("fix.lock");
334        match Self::create(&path) {
335            Ok(claim) => Ok(claim),
336            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
337                if Self::reclaim_if_dead(&path) {
338                    Self::create(&path).with_context(|| format!("lock {}", path.display()))
339                } else {
340                    bail!(
341                        "another `magi fix` is already running for this run ({} exists)",
342                        path.display()
343                    )
344                }
345            }
346            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
347        }
348    }
349
350    fn create(path: &Path) -> std::io::Result<Self> {
351        let mut f = std::fs::OpenOptions::new()
352            .write(true)
353            .create_new(true)
354            .open(path)?;
355        use std::io::Write as _;
356        // Read back by `reclaim_if_dead` on a later, stuck invocation.
357        writeln!(f, "{}", std::process::id())?;
358        Ok(Self {
359            path: path.to_owned(),
360        })
361    }
362
363    /// True if the lock named a process confirmed dead, in which case it was
364    /// also removed. Never true on an unreadable file, an unparsable pid, or
365    /// a liveness query the platform cannot answer — see this type's own doc.
366    fn reclaim_if_dead(path: &Path) -> bool {
367        let dead = std::fs::read_to_string(path)
368            .ok()
369            .and_then(|body| body.trim().parse::<u32>().ok())
370            .is_some_and(|pid| !crate::proc::pid_alive(pid));
371        dead && std::fs::remove_file(path).is_ok()
372    }
373}
374
375impl Drop for FixClaim {
376    fn drop(&mut self) {
377        let _ = std::fs::remove_file(&self.path);
378    }
379}
380
381impl Runner {
382    /// Start a fresh run against `repo`.
383    pub async fn start(repo: &Path, instruction: String, config: Config) -> Result<Self> {
384        let repo = git::toplevel(repo).await?;
385        let missing = agent::missing_programs(&config.agents);
386        if !missing.is_empty() {
387            bail!(
388                "these agent programs are not on PATH: {}. Fix the roster in \
389                 magi.toml or install them.",
390                missing.join(", ")
391            );
392        }
393        let base_branch = match config.merge.base.clone() {
394            Some(b) => b,
395            None => git::current_branch(&repo)
396                .await?
397                .context("HEAD is detached; set [merge] base in magi.toml")?,
398        };
399        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
400        // Still worth saying out loud. The operator's uncommitted work is not
401        // part of this run, and someone watching a candidate fail to use a
402        // change they just made deserves to know why.
403        if !git::is_clean(&repo).await? {
404            tracing::warn!(
405                "{} has uncommitted changes; they are not part of this run, \
406                 which branches off {base_branch} ({})",
407                repo.display(),
408                &base_commit[..base_commit.len().min(8)]
409            );
410        }
411        let roles = config.resolve_roles()?;
412        let max_parallel = config.graph.max_parallel.max(1);
413        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
414        state.event("start", format!("run {} created", state.id));
415        state.save()?;
416        Ok(Self {
417            state,
418            roles,
419            sem: Arc::new(Semaphore::new(max_parallel)),
420            pause: Pause::new(),
421            interrupt: Pause::new(),
422        })
423    }
424
425    /// Open a review-only run against work that already exists on `branch`.
426    ///
427    /// The expensive half of the graph is the implement wave — measured at
428    /// 111 and 134 internal tool-loop turns on this repository, against a
429    /// handful for a judge or a reviewer. The cheap half is worth running on
430    /// hand-written work too, and there was no way to reach it.
431    ///
432    /// No new state and no schema change are needed: a run with **one** viable
433    /// candidate and a tally already decided degrades `execute` to exactly
434    /// review → gate → merge, because `judge` skips a single-candidate field,
435    /// `deliberate` has fewer than two first choices to reconcile, `vote`
436    /// returns early, `tally` is already present and `fold_losers` has no
437    /// losers. Resuming such a run therefore does the right thing as well.
438    pub async fn review(repo: &Path, branch: &str, config: Config) -> Result<Self> {
439        let repo = git::toplevel(repo).await?;
440        let missing = agent::missing_programs(&config.agents);
441        if !missing.is_empty() {
442            bail!(
443                "these agent programs are not on PATH: {}. Fix the roster in \
444                 magi.toml or install them.",
445                missing.join(", ")
446            );
447        }
448        if !git::branch_exists(&repo, branch).await? {
449            bail!("no branch `{branch}` in {}", repo.display());
450        }
451        let base_branch = match config.merge.base.clone() {
452            Some(b) => b,
453            None => git::current_branch(&repo)
454                .await?
455                .context("HEAD is detached; set [merge] base in magi.toml")?,
456        };
457        if base_branch == branch {
458            bail!("`{branch}` is the base branch; there is nothing to review against");
459        }
460        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
461
462        let roles = config.resolve_roles()?;
463        let max_parallel = config.graph.max_parallel.max(1);
464        // The commit subjects are the closest thing to a task statement that
465        // existing work carries, and the reviewers are told as much.
466        let log = git::log_oneline(&repo, &base_commit, branch)
467            .await
468            .unwrap_or_default();
469        let instruction = format!(
470            "Review the work already on branch `{branch}`. There is no task \
471             statement: what the change claims to do is whatever its commits \
472             say.\n\n{}",
473            if log.trim().is_empty() {
474                "(no commit messages)"
475            } else {
476                log.trim()
477            }
478        );
479        let mut state = RunState::new(
480            repo.clone(),
481            base_branch,
482            base_commit.clone(),
483            instruction,
484            config,
485        );
486
487        // An attached worktree, so the fixer's commits land on the branch under
488        // review rather than on a detached head nobody will look at again.
489        let worktree = state.worktree_root().join("under-review");
490        if let Some(parent) = worktree.parent() {
491            tokio::fs::create_dir_all(parent).await.ok();
492        }
493        let path = worktree.to_string_lossy().to_string();
494        git::git(&repo, &["worktree", "add", &path, branch])
495            .await
496            .with_context(|| {
497                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
498            })?;
499
500        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
501            .await
502            .unwrap_or(0);
503        if commits == 0 {
504            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
505        }
506        let files = git::changed_files(&worktree, &base_commit, "HEAD")
507            .await
508            .map(|f| f.len())
509            .unwrap_or(0);
510        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
511            .await
512            .unwrap_or_default();
513
514        state.candidates.push(Candidate {
515            index: 0,
516            label: 'A',
517            // Not an agent id on purpose: nothing in the roster wrote this, and
518            // the stats tables must not credit anyone with a win for it.
519            agent: "(existing branch)".to_owned(),
520            branch: branch.to_owned(),
521            worktree,
522            summary: String::new(),
523            stat,
524            files,
525            commits,
526            empty: false,
527            failed: None,
528            verified_noop: None,
529            duration_ms: 0,
530            folded: false,
531        });
532        state.tally = Some(Tally {
533            first_choice: BTreeMap::from([('A', 0)]),
534            borda: BTreeMap::new(),
535            winner: 'A',
536            rankings: 0,
537            unanimous_initial: false,
538            deliberated: false,
539            changed_votes: 0,
540            unanimous_final: false,
541            tie_break: None,
542            // No panel sat, so no quorum applies. Zero judges is the correct
543            // number for work that never competed, and must not be reported as
544            // a collapsed panel.
545            judges: 0,
546            present: 0,
547            quorum: 0,
548            met_quorum: true,
549            uncontested: Some("review-only run: nothing competed".to_owned()),
550        });
551        state.status = RunStatus::Reviewing;
552        state.event(
553            "start",
554            format!(
555                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
556                state.id
557            ),
558        );
559        state.save()?;
560        Ok(Self {
561            state,
562            roles,
563            sem: Arc::new(Semaphore::new(max_parallel)),
564            pause: Pause::new(),
565            interrupt: Pause::new(),
566        })
567    }
568
569    /// Reopen an existing run.
570    pub fn resume(id: &str) -> Result<Self> {
571        let state = RunState::load(id)?;
572        let roles = state.config.resolve_roles()?;
573        let max_parallel = state.config.graph.max_parallel.max(1);
574        Ok(Self {
575            state,
576            roles,
577            sem: Arc::new(Semaphore::new(max_parallel)),
578            pause: Pause::new(),
579            interrupt: Pause::new(),
580        })
581    }
582
583    /// Walk the graph to a terminal state, skipping nodes already recorded.
584    pub async fn execute(&mut self) -> Result<()> {
585        // Moving again, so it is no longer parked. Set before the walk rather
586        // than in `resume`, so every way of re-entering the graph clears it
587        // and a card cannot claim a run is waiting to be resumed while the
588        // agents are already working.
589        self.state.parked = false;
590        // Any seat this state still lists as answering belongs to whatever
591        // process last drove this run — this one included, if it crashed
592        // mid-wave. Cleared and flushed immediately, before anything else
593        // runs, so a resume can never show a seat as live when nothing is
594        // asking it anything yet; the node that actually dispatches the next
595        // wave repopulates it.
596        self.state.clear_active();
597        // Recorded in the same spot, and flushed together with the clear
598        // above: this is the pid a reader checks (`RunState::liveness`) when
599        // no daemon claim exists to answer "is a process still driving this
600        // run" — a plain `magi run` / `magi review` typed into a terminal
601        // claims nothing there. Always overwritten, never only-if-absent, so
602        // a resumed run's stale pid from a previous, possibly-dead process
603        // can never survive into this one's own report. Unlike
604        // `clear_active`, this changes on every single `execute()` call, so
605        // the save below is now unconditional rather than only-if-cleared.
606        //
607        // `driver_started_at` is recorded in the same breath, from this same
608        // pid, so `liveness` can tell a live pid that is genuinely still us
609        // apart from one the OS has since handed to an unrelated process —
610        // see that field's own doc for why the pid alone is not enough.
611        let pid = std::process::id();
612        self.state.driver_pid = Some(pid);
613        self.state.driver_started_at = crate::proc::process_started_at(pid);
614        self.state.save()?;
615        // A run that already lost its quorum never resumes into the verdict
616        // machinery: `deliberate` and `vote` would otherwise clobber the
617        // stalled marker back to Voting and the run would keep going past a
618        // verdict that is no longer trustworthy. Everything already recorded is
619        // kept, so the run stays resumable (or foldable) for a human to pick up.
620        //
621        // On --resume the run gets one chance to repair itself: the seats a
622        // rate limit took out are re-asked. If their quota has since reset and
623        // the quorum is restored, the run picks up and finishes; otherwise it
624        // stays stale and still-resumable for a later retry. If it does not
625        // recover, the returned status stays `Stalled` and nothing was
626        // clobbered (the recovery only mutates entries for the lost seats).
627        if self.state.status == RunStatus::Stalled {
628            if self.recover_stall().await? {
629                self.finish_after_tally().await?;
630            } else {
631                // Still below quorum: persist the marker and stay resumable.
632                self.state.save()?;
633            }
634            return Ok(());
635        }
636        // A run parked inside `land` - watching CI, mid fix-round, or
637        // waiting on the owner's merge approval - resumes directly into it,
638        // never back through `prep`. Everything before `merge` already
639        // concluded; that is the only way `status` reaches `Landing` in the
640        // first place. Re-walking `review_loop` first would also be actively
641        // wrong: its own status recomputation (see its doc) treats any
642        // clean round as reason to set `status` to `Gating`, which would
643        // clobber this marker before `merge` ever ran, and this run would
644        // never find its way back into `land` at all.
645        if self.state.status == RunStatus::Landing {
646            self.run_land().await?;
647            // `run_land` may have settled the run right here - CI came back
648            // green and the PR merged, say - without ever passing back
649            // through `merge`'s own trailing call. Whatever it left `status`
650            // as is what this has to read.
651            self.settle_questions();
652            return Ok(());
653        }
654        self.prep().await?;
655        if self.park_here()? {
656            return Ok(());
657        }
658        self.advise().await?;
659        if self.park_here()? {
660            return Ok(());
661        }
662        self.implement().await?;
663        if self.park_here()? {
664            return Ok(());
665        }
666        // `after_implement` already saved the state and settled any open
667        // questions when it set this; nothing later in the graph has
668        // anything to judge.
669        if self.state.status == RunStatus::VerifiedNoop {
670            return Ok(());
671        }
672        self.judge().await?;
673        if self.park_here()? {
674            return Ok(());
675        }
676        self.deliberate().await?;
677        if self.park_here()? {
678            return Ok(());
679        }
680        self.vote().await?;
681        if self.park_here()? {
682            return Ok(());
683        }
684        self.tally()?;
685        // A verdict that lost its quorum is not trustworthy: do not review,
686        // gate, or merge on it. Everything already done is kept, so the run
687        // stays resumable (or foldable); the human can replace the agent that
688        // ran out of quota and pick it up.
689        if self.state.status == RunStatus::Stalled {
690            // Persist the stalled marker now — the normal end-of-execute save
691            // below is below this early return, and without it a resumed run
692            // would reload a pre-tally status and keep going.
693            self.state.save()?;
694            return Ok(());
695        }
696        self.finish_after_tally().await?;
697        Ok(())
698    }
699
700    /// Park here if asked to, recording it in the run's own timeline.
701    ///
702    /// Returns whether the caller should stop walking the graph. The state is
703    /// saved either way by the node that just finished; this adds the event so
704    /// the operator's card says why a run that is neither finished nor moving
705    /// is sitting where it is.
706    fn park_here(&mut self) -> Result<bool> {
707        // Either handle asking is enough - see `Pause`'s own doc for why
708        // they are never the same one. `interrupt` is checked second so a
709        // reason it carries is preferred in the message below over a plain
710        // shutdown park racing it at the same boundary.
711        if !self.pause.parked() && !self.interrupt.parked() {
712            return Ok(false);
713        }
714        let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
715            Some(reason) => format!(
716                "parked after `{}` ({reason}) — resume to carry on from here",
717                self.state.status.as_str()
718            ),
719            None => format!(
720                "parked after `{}` — resume to carry on from here",
721                self.state.status.as_str()
722            ),
723        };
724        self.state.event("park", why);
725        self.state.parked = true;
726        self.state.save()?;
727        Ok(true)
728    }
729
730    /// Hand the runner the pause `magi serve`'s own shutdown watches.
731    pub fn on_pause(&mut self, pause: Pause) {
732        self.pause = pause;
733    }
734
735    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
736    /// scheduler asking this one run - and no other - to park so a task
737    /// marked [`crate::queue::Task::interrupt`] can run alone. See
738    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
739    /// handle.
740    pub fn watch_interrupt(&mut self, pause: Pause) {
741        self.interrupt = pause;
742    }
743
744    /// Abandon this run's own open questions, once `status` has actually
745    /// settled rather than merely paused.
746    ///
747    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
748    /// either back up with the candidates, the review round and the seat
749    /// sessions already on disk, so a question an implementer asked mid-round
750    /// may still get a real answer read by a real resume. Only the statuses
751    /// `resumable` excludes are actually final: the run merged, it reached
752    /// `Ready` with nothing left to do, it failed outright with no
753    /// established point to continue from, or every candidate agreed, with
754    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
755    /// every one of those the seat that asked is gone for good, exactly like
756    /// the run being deleted under `magi run rm` - so the same cleanup
757    /// applies, worded for what actually happened instead of "the run was
758    /// deleted".
759    ///
760    /// Best-effort and silent on success: called from every place `status`
761    /// can land on one of those three, including ones a resumed run revisits,
762    /// so it must cost nothing when there was nothing open to begin with.
763    fn settle_questions(&mut self) {
764        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
765            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
766        }
767    }
768
769    /// The tail of the graph after a trustworthy tally: fold losers, review,
770    /// gate, merge, and persist.
771    async fn finish_after_tally(&mut self) -> Result<()> {
772        self.fold_losers().await?;
773        // Before review starts, and again right before the gate: a run's
774        // review rounds can themselves take long enough for the base to move
775        // a second time, and the gate is the one node whose "green" gets
776        // acted on.
777        self.sync_to_base().await?;
778        self.review_loop().await?;
779        self.sync_to_base().await?;
780        self.gate().await?;
781        self.merge().await?;
782        self.state.save()?;
783        Ok(())
784    }
785
786    // ---------------------------------------------------------------- prep
787
788    async fn prep(&mut self) -> Result<()> {
789        if !self.state.candidates.is_empty() {
790            return Ok(());
791        }
792        self.state.status = RunStatus::Prep;
793        let repo = self.state.repo.clone();
794        let base = self.state.base_commit.clone();
795        let root = self.state.worktree_root();
796        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
797
798        // The hook is the write-time half of the blindness contract; the
799        // presentation filter in `blind` is the half that cannot be bypassed.
800        let hooks_dir = self.state.dir().join("hooks");
801        if self.state.config.blind.commit_msg_hook {
802            std::fs::create_dir_all(&hooks_dir)
803                .with_context(|| format!("create {}", hooks_dir.display()))?;
804            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
805            let path = hooks_dir.join("commit-msg");
806            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
807            make_executable(&path)?;
808            // Ref-counted rather than a plain idempotent set: with more than
809            // one run able to be in flight in the same repository at once
810            // (see `Config::daemon.max_concurrent_runs`), a bare "already
811            // true?" check cannot tell "another run of mine still needs
812            // this" from "nobody does", and the run that happens to finish
813            // first would disable the hook out from under a sibling still
814            // relying on it.
815            git::acquire_worktree_config(&repo).await?;
816            self.state.enabled_worktree_config = true;
817        }
818
819        for (index, (spec, label)) in self
820            .roles
821            .implementers
822            .clone()
823            .into_iter()
824            .zip(labels)
825            .enumerate()
826        {
827            let branch = self.state.branch_for(label);
828            let worktree = root.join(format!("cand-{label}"));
829            git::worktree_add_branch(&repo, &worktree, &branch, &base).await?;
830            if self.state.config.blind.commit_msg_hook {
831                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
832            }
833            git::local_exclude(&worktree, "/.magi/").await?;
834            self.state.candidates.push(Candidate {
835                index,
836                label,
837                agent: spec.id.clone(),
838                branch,
839                worktree,
840                summary: String::new(),
841                stat: String::new(),
842                files: 0,
843                commits: 0,
844                empty: false,
845                failed: None,
846                verified_noop: None,
847                duration_ms: 0,
848                folded: false,
849            });
850        }
851
852        for j in 1..=self.roles.judges.len() {
853            let wt = root.join(format!("judge-{j}"));
854            if !wt.exists() {
855                git::worktree_add_detached(&repo, &wt, &base).await?;
856            }
857        }
858
859        // Disposable, detached checkouts for the design-deliberation stage's
860        // advisor seats — the same shape as the judges' above, at the same
861        // base commit, since advisors also only ever read. Sized off the
862        // configured count directly rather than a resolved roster: unlike
863        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
864        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
865        // `prep` has no `ResolvedRoles` field to read a count from here.
866        if self.state.config.graph.advise {
867            for k in 1..=self.state.config.graph.advisors {
868                let wt = root.join(format!("advisor-{k}"));
869                if !wt.exists() {
870                    git::worktree_add_detached(&repo, &wt, &base).await?;
871                }
872            }
873        }
874
875        // A judge cannot tell it is looking at its own patch — the seats keep
876        // separate conversations — but a panel that shares agents with the
877        // field is less independent than it looks, and that is worth saying out
878        // loud once per run rather than leaving it in the config.
879        let authors: Vec<&str> = self
880            .roles
881            .implementers
882            .iter()
883            .map(|a| a.id.as_str())
884            .collect();
885        let overlap: Vec<String> = self
886            .roles
887            .judges
888            .iter()
889            .enumerate()
890            .filter(|(_, j)| authors.contains(&j.id.as_str()))
891            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
892            .collect();
893        if !overlap.is_empty() {
894            let note = format!(
895                "{} also authored a candidate; blind, but the panel is less \
896                 independent than {} distinct agents would be",
897                overlap.join(", "),
898                self.roles.judges.len()
899            );
900            self.state.event("prep", note);
901        }
902
903        self.state.event(
904            "prep",
905            format!(
906                "{} candidates, {} judges, base {} ({})",
907                self.state.candidates.len(),
908                self.roles.judges.len(),
909                &self.state.base_commit[..7.min(self.state.base_commit.len())],
910                self.state.base_branch
911            ),
912        );
913        self.state.status = RunStatus::Implementing;
914        self.state.save()?;
915        Ok(())
916    }
917
918    // -------------------------------------------------------------- advise
919
920    /// The design-deliberation stage: independent, read-only advisor seats
921    /// each sketch a design before any implementer touches the repository,
922    /// and (when at least one produced a usable proposal) a synthesis seat
923    /// blends them into a brief `implement` carries in every candidate's
924    /// prompt.
925    ///
926    /// `[graph] advise` is the on/off switch, on by default; `[graph]
927    /// advisors` is the proposal count. Everything here is best-effort and
928    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
929    /// that cannot reach quota, or a synthesis seat that produced nothing
930    /// usable all leave `implement` exactly as it was before this stage
931    /// existed — the task instruction alone — rather than failing the whole
932    /// competition over an enrichment stage. Every outcome is still recorded
933    /// as an event, so a run that got nothing from this stage says why.
934    ///
935    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
936    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
937    /// resumed run whose stage failed would re-run it, and re-spend the
938    /// agent calls, on every reentry before `implement`.
939    ///
940    /// Also skipped once any candidate shows implementation progress — the
941    /// exact predicate `implement` itself uses to decide a candidate is no
942    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
943    /// is not enough: a run created by an older binary that predates this
944    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
945    /// an already-`Implementing`-or-later run under this build would
946    /// otherwise walk straight back through `prep` (a no-op once candidates
947    /// exist) into this node and spawn every advisor seat against worktrees
948    /// `prep` never recreated — after implementation has already started,
949    /// which is exactly the invariant this stage exists to guarantee.
950    async fn advise(&mut self) -> Result<()> {
951        let implement_untouched = self
952            .state
953            .candidates
954            .iter()
955            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
956        if !self.state.config.graph.advise || self.state.advise_attempted {
957            return Ok(());
958        }
959        if !implement_untouched {
960            self.state.event(
961                "advise",
962                "skipping the design-deliberation stage: at least one \
963                 candidate already shows implementation progress, so this \
964                 run is past the point the stage exists to run before"
965                    .to_owned(),
966            );
967            self.state.advise_attempted = true;
968            self.state.save()?;
969            return Ok(());
970        }
971        let run_id = self.state.id.clone();
972        let prompts = self.state.config.prompts.clone();
973        let instruction = self.state.instruction.clone();
974        let language = self.state.config.graph.language.clone();
975        let root = self.state.worktree_root();
976        let n = self.state.config.graph.advisors;
977        let where_recorded = self.state.dir().join("run.json");
978
979        let seats = match self.state.config.advisors() {
980            Ok(seats) if !seats.is_empty() => seats,
981            Ok(_) => {
982                self.state.event(
983                    "advise",
984                    format!(
985                        "[graph] advisors is 0; skipping the design-deliberation \
986                         stage and continuing without a synthesis brief (see {})",
987                        where_recorded.display()
988                    ),
989                );
990                self.state.advise_attempted = true;
991                self.state.save()?;
992                return Ok(());
993            }
994            Err(e) => {
995                self.state.event(
996                    "advise",
997                    format!(
998                        "could not resolve advisor seats ({e:#}); continuing \
999                         without a design-deliberation brief (see {})",
1000                        where_recorded.display()
1001                    ),
1002                );
1003                self.state.advise_attempted = true;
1004                self.state.save()?;
1005                return Ok(());
1006            }
1007        };
1008
1009        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1010        let artifacts = agent::artifacts_dir(&self.state.dir());
1011        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1012
1013        let mut jobs = Vec::new();
1014        for (i, spec) in seats.iter().cloned().enumerate() {
1015            let seat_key = format!("advisor-{}", i + 1);
1016            let seat = self.seat(&seat_key, &spec.id);
1017            jobs.push(SeatJob {
1018                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1019                spec,
1020                seat,
1021                cwd: worktrees[i % worktrees.len()].clone(),
1022                timeout,
1023                allow_write: false,
1024                sessions: false,
1025                artifacts: artifacts.clone(),
1026                stem: seat_key,
1027            });
1028        }
1029
1030        self.state.event(
1031            "advise",
1032            format!(
1033                "{} advisor seat(s) sketching a design in parallel",
1034                jobs.len()
1035            ),
1036        );
1037        let mut quota_losses = Vec::new();
1038        let cache = self.state.config.cache_dir();
1039        let ctx = WaveCtx {
1040            run: &run_id,
1041            node: "advise",
1042            prompts: &prompts,
1043            cache: cache.as_deref(),
1044            round: None,
1045        };
1046        let results = ask_json_wave::<Proposal>(
1047            jobs,
1048            Arc::clone(&self.sem),
1049            self.state.config.graph.retries,
1050            &ctx,
1051            &mut quota_losses,
1052            &mut self.state,
1053            &|p: &Proposal| p.validate(),
1054        )
1055        .await;
1056        self.state.quota.extend(quota_losses);
1057
1058        let mut records = Vec::with_capacity(results.len());
1059        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1060            let agent_id = seat.agent.clone();
1061            self.state.seats.insert(seat.key.clone(), seat);
1062            match res {
1063                Ok((proposal, out)) => {
1064                    self.state
1065                        .event("advise", format!("advisor-{} proposed a design", i + 1));
1066                    records.push(advise::AdvisorRecord::proposed(
1067                        i + 1,
1068                        agent_id,
1069                        proposal,
1070                        out.duration_ms,
1071                    ));
1072                }
1073                Err(e) => {
1074                    self.state.event(
1075                        "advise",
1076                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1077                    );
1078                    records.push(advise::AdvisorRecord::failed(
1079                        i + 1,
1080                        agent_id,
1081                        e.to_string(),
1082                    ));
1083                }
1084            }
1085        }
1086
1087        let mut advice = advise::Advice {
1088            records,
1089            synthesis: None,
1090        };
1091        if advice.proposals().is_empty() {
1092            self.state.event(
1093                "advise",
1094                "no advisor produced a usable proposal; continuing without a \
1095                 synthesis brief"
1096                    .to_owned(),
1097            );
1098        } else {
1099            match self
1100                .synthesize_brief(
1101                    &advice,
1102                    &instruction,
1103                    &language,
1104                    &worktrees[0],
1105                    &artifacts,
1106                    &run_id,
1107                    &prompts,
1108                    cache.as_deref(),
1109                )
1110                .await
1111            {
1112                Ok(Some(text)) => {
1113                    self.state.event(
1114                        "advise",
1115                        "synthesized a design brief for the implementer".to_owned(),
1116                    );
1117                    advice.synthesis = Some(text);
1118                }
1119                Ok(None) => {
1120                    self.state.event(
1121                        "advise",
1122                        "the synthesis seat produced nothing usable; continuing \
1123                         without a design brief"
1124                            .to_owned(),
1125                    );
1126                }
1127                Err(e) => {
1128                    self.state.event(
1129                        "advise",
1130                        format!("could not synthesize a design brief: {e:#}"),
1131                    );
1132                }
1133            }
1134        }
1135        advise::apply_reflection(&mut advice);
1136
1137        self.state.advice = Some(advice);
1138        self.state.advise_attempted = true;
1139        self.state.save()?;
1140        Ok(())
1141    }
1142
1143    /// The synthesis seat: reads every advisor's proposal and blends them
1144    /// into the design brief `advise` stores on [`RunState::advice`]. Split
1145    /// out of [`Runner::advise`] only for readability — it is not called
1146    /// anywhere else.
1147    ///
1148    /// Picked the same way [`crate::talk`]'s standing conversation and
1149    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
1150    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
1151    /// order (a claude seat, else the first runnable agent in roster order)
1152    /// used when that field is unset — see `[roles] synthesizer`'s own doc
1153    /// in [`crate::config`] for why a dedicated field exists here at all.
1154    #[allow(clippy::too_many_arguments)]
1155    async fn synthesize_brief(
1156        &mut self,
1157        advice: &advise::Advice,
1158        instruction: &str,
1159        language: &str,
1160        cwd: &Path,
1161        artifacts: &Path,
1162        run_id: &str,
1163        prompts: &Prompts,
1164        cache: Option<&Path>,
1165    ) -> Result<Option<String>> {
1166        let want = self.state.config.roles.synthesizer.as_deref();
1167        let spec = agent::pick(&self.state.config.agents, want, &agent::installed)?;
1168        let mut seat = self.seat("advise-synthesis", &spec.id);
1169        let proposals = advice.proposals();
1170        let mut prompt = prompt::with_overlay(
1171            prompt::synthesize_brief(instruction, &proposals, language),
1172            prompts.overlay("advise"),
1173        );
1174        if cache.is_some() {
1175            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
1176            // below — see `prompt::build_cache_note`'s doc for why telling a
1177            // read-only seat to build through the shared cache is exactly how
1178            // a sandbox's write refusal gets misread as a defect.
1179            prompt.push('\n');
1180            prompt.push_str(&prompt::build_cache_note("advise", false));
1181        }
1182        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1183        let out = agent::invoke(
1184            &spec,
1185            &mut seat,
1186            &Invocation {
1187                cwd,
1188                prompt: &prompt,
1189                timeout,
1190                allow_write: false,
1191                sessions: false,
1192                artifacts,
1193                stem: "advise-synthesis",
1194                run: run_id,
1195                node: "advise",
1196                cache_dir: None,
1197                attachments: &[],
1198            },
1199        )
1200        .await?;
1201        self.state.seats.insert(seat.key.clone(), seat);
1202        if !out.usable() {
1203            return Ok(None);
1204        }
1205        let text =
1206            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1207        Ok((!text.trim().is_empty()).then_some(text))
1208    }
1209
1210    // ----------------------------------------------------------- implement
1211
1212    async fn implement(&mut self) -> Result<()> {
1213        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1214        // agent files with `magi task add` name the run that paid for it. The
1215        // prompt overlay is cloned alongside it because the waves borrow it
1216        // while `self` is mutably borrowed by the node's own bookkeeping.
1217        let run_id = self.state.id.clone();
1218        let prompts = self.state.config.prompts.clone();
1219        let todo: Vec<usize> = self
1220            .state
1221            .candidates
1222            .iter()
1223            .enumerate()
1224            .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1225            .map(|(i, _)| i)
1226            .collect();
1227        if todo.is_empty() {
1228            return self.after_implement();
1229        }
1230        self.state.status = RunStatus::Implementing;
1231
1232        let language = self.state.config.graph.language.clone();
1233        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1234        let sessions = self.state.config.graph.sessions;
1235        let artifacts = agent::artifacts_dir(&self.state.dir());
1236        // The design-deliberation stage's blended brief, when `advise` found
1237        // one — carried into every implementer's prompt the same way
1238        // regardless of which candidate it is.
1239        let brief = self
1240            .state
1241            .advice
1242            .as_ref()
1243            .and_then(|a| a.synthesis.as_deref())
1244            .map(str::to_owned);
1245
1246        let mut jobs = Vec::new();
1247        for &i in &todo {
1248            let (index, label, worktree) = {
1249                let c = &self.state.candidates[i];
1250                (c.index, c.label, c.worktree.clone())
1251            };
1252            let spec = self.roles.implementers[index].clone();
1253            let seat_key = format!("impl-{label}");
1254            let seat = self.seat(&seat_key, &spec.id);
1255            let instruction = self.state.instruction.clone();
1256            jobs.push(SeatJob {
1257                spec,
1258                seat,
1259                prompt: prompt::implement(
1260                    &instruction,
1261                    &worktree.to_string_lossy(),
1262                    &language,
1263                    brief.as_deref(),
1264                ),
1265                cwd: worktree,
1266                timeout,
1267                allow_write: true,
1268                sessions,
1269                artifacts: artifacts.clone(),
1270                stem: format!("impl-{label}"),
1271            });
1272        }
1273
1274        self.state.event(
1275            "implement",
1276            format!("{} candidates in parallel", jobs.len()),
1277        );
1278        // Kept so a seat whose CLI hung up can be asked again from the same
1279        // job: `wave` consumes what it is given. Mutable so `resume_quota_losses`
1280        // can update a seat's own entry once a fallback agent takes it over —
1281        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
1282        // whichever agent actually answered, not the one that quota'd out.
1283        let mut sent = jobs.clone();
1284        let cache = self.state.config.cache_dir();
1285        let ctx = WaveCtx {
1286            run: &run_id,
1287            node: "implement",
1288            prompts: &prompts,
1289            cache: cache.as_deref(),
1290            round: None,
1291        };
1292        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1293        self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1294            .await;
1295        self.resume_quota_losses(&mut results, &mut sent, &prompts, &run_id)
1296            .await;
1297        self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1298            .await;
1299
1300        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1301            let seat_key = seat.key.clone();
1302            // A quota fallback (`resume_quota_losses`) may have handed this
1303            // seat to a different agent than the one `prep` recorded on the
1304            // candidate; the stats tables and any later fixer-defaults-to-
1305            // winner's-author lookup must credit whoever actually answered —
1306            // unless every fallback also quota'd out, in which case nobody
1307            // actually answered and crediting the last agent tried would
1308            // erase every earlier agent's own quota loss from the stats
1309            // tables instead of just this one seat's.
1310            let agent = seat.agent.clone();
1311            let exhausted_the_fallback_chain = matches!(&out, AgentOutcome::Quota(_));
1312            self.state.seats.insert(seat.key.clone(), seat);
1313            let label = self.state.candidates[i].label;
1314            let worktree = self.state.candidates[i].worktree.clone();
1315            let base = self.state.base_commit.clone();
1316
1317            let (summary, duration, failed, verified_claim) = match out {
1318                AgentOutcome::Ok(o) => {
1319                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1320                    let failed = (!o.usable()).then(|| {
1321                        if o.timed_out {
1322                            "agent timed out".to_owned()
1323                        } else {
1324                            format!("agent exited with {:?}", o.exit_code)
1325                        }
1326                    });
1327                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
1328                    (text, o.duration_ms, failed, verified_claim)
1329                }
1330                // Left un-resumed by `resume_undelivered` (a dirty tree
1331                // already rescues the work, or there was no session left to
1332                // resume into) — reported like the ordinary failure it is,
1333                // never as if `o.text` (the CLI's raw error JSON) were an
1334                // answer.
1335                AgentOutcome::Dropped(o) => {
1336                    let why = o
1337                        .dropped
1338                        .as_ref()
1339                        .map(|d| d.why.as_str())
1340                        .unwrap_or("the CLI ended the stream without delivering its answer");
1341                    (
1342                        String::new(),
1343                        o.duration_ms,
1344                        Some(format!("the CLI dropped the stream ({why})")),
1345                        None,
1346                    )
1347                }
1348                AgentOutcome::Quota(o) => {
1349                    self.state.quota.push(QuotaLoss {
1350                        seat: seat_key,
1351                        node: "implement".to_owned(),
1352                        at: Timestamp::now(),
1353                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1354                    });
1355                    (
1356                        String::new(),
1357                        o.duration_ms,
1358                        Some("rate limited (quota); produced no change".to_owned()),
1359                        None,
1360                    )
1361                }
1362                AgentOutcome::Failed(e) => (String::new(), 0, Some(e), None),
1363            };
1364
1365            // Rescue anything the agent edited but never committed: an
1366            // uncommitted candidate would silently be an empty one.
1367            let rescued = git::commit_all(
1368                &worktree,
1369                &format!("magi: candidate {label} (uncommitted work)"),
1370            )
1371            .await
1372            .unwrap_or(false);
1373            let commits = git::commits_ahead(&worktree, &base, "HEAD")
1374                .await
1375                .unwrap_or(0);
1376            let patch = git::diff(&worktree, &base, "HEAD")
1377                .await
1378                .unwrap_or_default();
1379            let stat = git::diff_stat(&worktree, &base, "HEAD")
1380                .await
1381                .unwrap_or_default();
1382            let files = git::changed_files(&worktree, &base, "HEAD")
1383                .await
1384                .map(|f| f.len())
1385                .unwrap_or(0);
1386            write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1387
1388            let c = &mut self.state.candidates[i];
1389            if !exhausted_the_fallback_chain {
1390                c.agent = agent;
1391            }
1392            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1393            c.stat = stat;
1394            c.files = files;
1395            c.commits = commits;
1396            c.duration_ms = duration;
1397            c.empty = commits == 0 || patch.trim().is_empty();
1398            // An agent that failed but still produced a committed change stays
1399            // in the running: the patch is what gets judged, not the exit code.
1400            c.failed = match failed {
1401                Some(_) if c.empty => failed,
1402                _ => None,
1403            };
1404            // Only an empty candidate can be a verified no-op: a claim next
1405            // to a real patch is not what the marker is for, and `c.failed`
1406            // being `Some` here already implies `verified_claim` was never
1407            // set (see the guard above the match that produced it).
1408            c.verified_noop = if c.empty { verified_claim } else { None };
1409            let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
1410                (Some(e), _, _, _) => format!("candidate {label}: {e}"),
1411                (None, true, Some(_), _) => {
1412                    format!("candidate {label}: no change produced (agent-verified no-op)")
1413                }
1414                (None, true, None, _) => format!("candidate {label}: no change produced"),
1415                (None, false, _, true) => {
1416                    format!(
1417                        "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1418                    )
1419                }
1420                (None, false, _, false) => {
1421                    format!("candidate {label}: {files} files, {commits} commits")
1422                }
1423            };
1424            self.state.event("implement", note);
1425            self.state.save()?;
1426        }
1427
1428        self.after_implement()
1429    }
1430
1431    /// Ask again, once, for work a CLI did and then failed to hand over.
1432    ///
1433    /// [`agent::dropped_stream`] recognises the one shape observed: an error
1434    /// status with an empty response and a usage report showing output tokens,
1435    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
1436    /// seven minutes and 14,267 output tokens that arrived as an empty
1437    /// candidate, because `agy`'s own subscriber fell behind and hung up.
1438    ///
1439    /// Two conditions, and both matter:
1440    ///
1441    /// - **Only when the tree is untouched.** Often the agent has already
1442    ///   written its files and only the closing message was lost; the rescue
1443    ///   commit below picks that up and there is nothing to ask for. Re-asking
1444    ///   then would pay for a second implementation of work already on disk.
1445    /// - **Once.** A CLI that drops one stream can drop the next, and this
1446    ///   node is the most expensive in the graph.
1447    ///
1448    /// The re-ask is a resume, not a re-run: `has_context` is true because the
1449    /// dropped reply still carried its `conversation_id`, so the seat is asked
1450    /// to finish what it was doing rather than sent the whole task again. It
1451    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
1452    /// node's - for the same reason a re-ranked judge does: restating finished
1453    /// work is not the work.
1454    ///
1455    /// Unlike a quota this is worth retrying at all: a rate limit fails the
1456    /// same way until it resets, while an abandoned conversation is still
1457    /// there to be picked up.
1458    async fn resume_undelivered(
1459        &mut self,
1460        results: &mut [(usize, SeatState, AgentOutcome)],
1461        sent: &[SeatJob],
1462        prompts: &Prompts,
1463        run_id: &str,
1464    ) {
1465        for (wi, seat, out) in results.iter_mut() {
1466            let Some(dropped) = (match &*out {
1467                AgentOutcome::Dropped(o) => o.dropped.clone(),
1468                _ => None,
1469            }) else {
1470                continue;
1471            };
1472            let Some(job) = sent.get(*wi) else { continue };
1473            // Already on disk? Then only the closing message was lost.
1474            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1475                self.state.event(
1476                    "implement",
1477                    format!(
1478                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1479                         work is in the tree",
1480                        seat.key, dropped.output_tokens, dropped.why
1481                    ),
1482                );
1483                continue;
1484            }
1485            // The re-ask only makes sense as a resume: `resume_after_drop`
1486            // says nothing about the task, trusting the seat to still hold it.
1487            // Without a session to resume — sessions disabled, or this CLI's
1488            // drop shape happened not to carry a session id — that prompt
1489            // would open a brand-new conversation with no context at all,
1490            // which is worse than leaving this as the ordinary failure it
1491            // already is.
1492            if !has_context(&job.spec, seat, job.sessions) {
1493                self.state.event(
1494                    "implement",
1495                    format!(
1496                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1497                         is no session left to resume",
1498                        seat.key, dropped.output_tokens, dropped.why
1499                    ),
1500                );
1501                continue;
1502            }
1503            self.state.event(
1504                "implement",
1505                format!(
1506                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1507                     conversation",
1508                    seat.key, dropped.output_tokens, dropped.why
1509                ),
1510            );
1511            let mut retry = job.clone();
1512            retry.seat = seat.clone();
1513            retry.prompt = prompt::resume_after_drop(&dropped.why);
1514            retry.timeout = retry_budget(job.timeout, true);
1515            retry.stem = format!("{}-resume", job.stem);
1516            let cache = self.state.config.cache_dir();
1517            let ctx = WaveCtx {
1518                run: run_id,
1519                node: "implement",
1520                prompts,
1521                cache: cache.as_deref(),
1522                round: None,
1523            };
1524            let (resumed_seat, resumed) =
1525                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1526            *seat = resumed_seat;
1527            *out = resumed;
1528        }
1529    }
1530
1531    /// Fall an implement seat through to the next untried agent in the
1532    /// implementer roster when it lost to quota, instead of leaving the
1533    /// seat's loss final the moment one agent's account runs dry.
1534    ///
1535    /// Solo runs (`graph.candidates = 1`, `daemon::apply_solo`'s forced shape)
1536    /// are the motivating case: `Config::resolve_roles`'s `implementers`
1537    /// truncates to the single slot rotation picked, so a solo task whose one
1538    /// implementer hits quota mid-run used to have nothing else to try. This
1539    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
1540    /// unrotated roster — which is the only place the *other* candidates in
1541    /// the machine's roster still exist once `implementers` has been cut down
1542    /// to size.
1543    ///
1544    /// Walks forward from just past the seat's own original position in the
1545    /// roster, never wrapping back to the front: a later candidate slot (say
1546    /// `beta`, the roster's second entry) must fall through to the *next*
1547    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
1548    /// almost certainly a different candidate's own agent already — and once
1549    /// the roster's tail is exhausted there is nothing left to fall through
1550    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
1551    /// whole [`AgentSpec`]: a roster with the same id named twice must not
1552    /// let this retry that id forever. The loop keeps falling through until
1553    /// an attempt lands something other than `Quota` or the roster's tail
1554    /// runs out of untried ids, at which point the seat is left exactly as
1555    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
1556    /// `QuotaLoss` recorded, the candidate failed/empty.
1557    ///
1558    /// `sent` is taken mutably and updated with the fallback agent's spec:
1559    /// `resume_unconfirmed_commands`, which runs after this and also reads
1560    /// `sent`, must see whichever agent actually ended up answering the seat
1561    /// — reading the stale, original spec there would check session
1562    /// eligibility against the wrong CLI and could hand a fallback agent's
1563    /// session id to the agent that just lost the seat to quota.
1564    ///
1565    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
1566    /// — `self.seat` only reuses state when the agent id is unchanged, so
1567    /// handing it a different id already gets this for free. Reusing the old
1568    /// seat would resume a different CLI's session as if it were a
1569    /// continuation of this one.
1570    ///
1571    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
1572    /// a quota loss cuts an agent off mid-turn, so anything already in the
1573    /// tree is unfinished work, not a completed candidate a re-ask would pay
1574    /// for twice. A dirty tree is rescued into a commit first (the same
1575    /// neutral-identity rescue `implement`'s own outcome loop gives every
1576    /// candidate) so the next agent starts clean.
1577    ///
1578    /// The new agent gets the implementer's full prompt and full
1579    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
1580    /// it has no session and no context, and is implementing the task from
1581    /// nothing, unlike a resumed drop which is only restating work already
1582    /// done.
1583    ///
1584    /// Every intermediate `Quota` this loop absorbs is folded into a plain
1585    /// `implement` event, never into `self.state.quota` — that is what
1586    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
1587    /// go unspent, and a seat that ultimately recovered on its second or
1588    /// third agent is not the stalled panel that check exists to catch. Only
1589    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
1590    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
1591    /// outcome loop already has — this helper never pushes to it itself.
1592    async fn resume_quota_losses(
1593        &mut self,
1594        results: &mut [(usize, SeatState, AgentOutcome)],
1595        sent: &mut [SeatJob],
1596        prompts: &Prompts,
1597        run_id: &str,
1598    ) {
1599        let instruction = self.state.instruction.clone();
1600        let language = self.state.config.graph.language.clone();
1601        let brief = self
1602            .state
1603            .advice
1604            .as_ref()
1605            .and_then(|a| a.synthesis.as_deref())
1606            .map(str::to_owned);
1607        for (wi, seat, out) in results.iter_mut() {
1608            let Some(job) = sent.get_mut(*wi) else {
1609                continue;
1610            };
1611            // Where the seat's own original agent sits in the roster — the
1612            // fallback walk starts just past here, never at the front, so a
1613            // later candidate slot's quota loss does not fall back onto an
1614            // earlier slot's own agent.
1615            let start = self
1616                .roles
1617                .implementer_roster
1618                .iter()
1619                .position(|s| s.id == job.spec.id)
1620                .unwrap_or(0);
1621            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
1622            let mut fallback_attempt = 0usize;
1623            while matches!(&*out, AgentOutcome::Quota(_)) {
1624                let Some(next) =
1625                    next_untried_implementer(&self.roles.implementer_roster, start, &tried)
1626                        .cloned()
1627                else {
1628                    break;
1629                };
1630                tried.insert(next.id.clone());
1631                fallback_attempt += 1;
1632
1633                git::commit_all(
1634                    &job.cwd,
1635                    &format!(
1636                        "magi: candidate {} (uncommitted work before quota fallback)",
1637                        seat.key
1638                    ),
1639                )
1640                .await
1641                .ok();
1642
1643                self.state.event(
1644                    "implement",
1645                    format!(
1646                        "{}: rate limited (quota) on {}; retrying with {}",
1647                        seat.key, seat.agent, next.id
1648                    ),
1649                );
1650
1651                let new_seat = self.seat(&seat.key, &next.id);
1652                // Kept in sync on `sent` itself, not just the local retry: a
1653                // later helper (`resume_unconfirmed_commands`) reads `sent`
1654                // after this one returns and must see whichever agent is now
1655                // occupying the seat, not the one that just quota'd out —
1656                // otherwise it would judge session/continuation eligibility
1657                // by the wrong CLI and could resend a fallback's session id
1658                // to the agent that lost it the seat in the first place.
1659                job.spec = next.clone();
1660                let mut retry = job.clone();
1661                retry.seat = new_seat;
1662                retry.prompt = prompt::implement(
1663                    &instruction,
1664                    &job.cwd.to_string_lossy(),
1665                    &language,
1666                    brief.as_deref(),
1667                );
1668                retry.stem = format!("{}-quota-{}", job.stem, next.id);
1669                let cache = self.state.config.cache_dir();
1670                let ctx = WaveCtx {
1671                    run: run_id,
1672                    node: "implement",
1673                    prompts,
1674                    cache: cache.as_deref(),
1675                    round: None,
1676                };
1677                let (fallback_seat, fallback_out) = run_one(
1678                    retry,
1679                    Arc::clone(&self.sem),
1680                    &ctx,
1681                    &mut self.state,
1682                    fallback_attempt,
1683                )
1684                .await;
1685                *seat = fallback_seat;
1686                *out = fallback_out;
1687            }
1688        }
1689    }
1690
1691    /// Ask an implement seat's own CLI to confirm what it started, once, when
1692    /// its reply reported a command whose completion status it never
1693    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
1694    /// that does and does not mean.
1695    ///
1696    /// The completion contract this task asks for, extended to `implement`
1697    /// with the same signal `continue_fix_report` reads for the fixer,
1698    /// rather than a keyword search over the reply or a hard requirement on
1699    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
1700    /// a candidate's CLI turn ended cleanly while a test run it had started
1701    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
1702    /// named in it at all is untouched by this: `commands` is empty, so
1703    /// there is nothing to be unconfirmed.
1704    ///
1705    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
1706    /// this is not about recovering edits that might already be on disk, it
1707    /// is about a result the seat itself never vouched for, which resuming
1708    /// asks for regardless of what the tree already holds. Bounded to one
1709    /// attempt for the same reason `resume_undelivered` is — this is the
1710    /// most expensive node in the graph — and a seat that still cannot
1711    /// confirm on that attempt is left as whatever its (possibly still
1712    /// unconfirmed) reply says; this does not invent a new "failed" reason
1713    /// for a candidate that otherwise produced a real, committed change.
1714    async fn resume_unconfirmed_commands(
1715        &mut self,
1716        results: &mut [(usize, SeatState, AgentOutcome)],
1717        sent: &[SeatJob],
1718        prompts: &Prompts,
1719        run_id: &str,
1720    ) {
1721        for (wi, seat, out) in results.iter_mut() {
1722            let AgentOutcome::Ok(o) = &*out else {
1723                continue;
1724            };
1725            if !has_unconfirmed_command(&o.commands) {
1726                continue;
1727            }
1728            let Some(job) = sent.get(*wi) else { continue };
1729            if !has_context(&job.spec, seat, job.sessions) {
1730                self.state.event(
1731                    "implement",
1732                    format!(
1733                        "{}: the reply named a command whose own CLI never confirmed the exit \
1734                         status of, but there is no session left to resume",
1735                        seat.key
1736                    ),
1737                );
1738                continue;
1739            }
1740            self.state.event(
1741                "implement",
1742                format!(
1743                    "{}: the reply named a command whose own CLI never confirmed the exit \
1744                     status of; resuming the conversation",
1745                    seat.key
1746                ),
1747            );
1748            let mut retry = job.clone();
1749            retry.seat = seat.clone();
1750            retry.prompt = prompt::resume_incomplete(
1751                "a command in your last reply had no confirmed exit status",
1752            );
1753            retry.timeout = retry_budget(job.timeout, true);
1754            retry.stem = format!("{}-confirm", job.stem);
1755            let cache = self.state.config.cache_dir();
1756            let ctx = WaveCtx {
1757                run: run_id,
1758                node: "implement",
1759                prompts,
1760                cache: cache.as_deref(),
1761                round: None,
1762            };
1763            let (resumed_seat, resumed) =
1764                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1765            *seat = resumed_seat;
1766            *out = resumed;
1767        }
1768    }
1769
1770    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
1771    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
1772    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
1773    /// that motivated this.
1774    ///
1775    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
1776    /// own nudge loop already covers for judge/review/vote seats, and not a
1777    /// dropped stream, which [`Runner::resume_undelivered`] covers for
1778    /// implement seats: here the CLI turn genuinely finished while the node's
1779    /// own work — the fixer's account of what it did — had not. Gated purely
1780    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
1781    /// reply, never on any wording in it, so a fixer whose valid, first-try
1782    /// `FixReport` happens to mention having waited on a background test is
1783    /// never resumed — the `Ok(report)` branch at the call site returns
1784    /// before this is ever invoked.
1785    ///
1786    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
1787    /// attempt ([`retry_budget`]), nothing attempted once the session is
1788    /// gone, and a quota hit ends the loop immediately rather than retrying a
1789    /// rate limit that fails the same way again.
1790    async fn continue_fix_report(
1791        &mut self,
1792        mut seat: SeatState,
1793        parse_err: String,
1794        job: &SeatJob,
1795        prompts: &Prompts,
1796        run_id: &str,
1797        round: usize,
1798    ) -> (
1799        SeatState,
1800        Option<FixReport>,
1801        Option<String>,
1802        ContinuationRecord,
1803    ) {
1804        let mut last_err = parse_err;
1805        let mut cumulative_wait_ms = 0u64;
1806        let mut attempts = 0usize;
1807        loop {
1808            if !has_context(&job.spec, &seat, job.sessions) {
1809                self.state.event(
1810                    "fix",
1811                    format!(
1812                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
1813                         session left to resume into"
1814                    ),
1815                );
1816                let outcome = if attempts == 0 {
1817                    ContinuationOutcome::NoSession
1818                } else {
1819                    ContinuationOutcome::Exhausted
1820                };
1821                return (
1822                    seat,
1823                    None,
1824                    Some(format!("unparsable fix report: {last_err}")),
1825                    ContinuationRecord {
1826                        attempts,
1827                        cumulative_wait_ms,
1828                        outcome,
1829                    },
1830                );
1831            }
1832            if attempts >= MAX_FIX_CONTINUATIONS {
1833                self.state.event(
1834                    "fix",
1835                    format!(
1836                        "round {round}: fixer's reply still had no adoption report after \
1837                         {attempts} continuation(s) ({last_err}); giving up"
1838                    ),
1839                );
1840                return (
1841                    seat,
1842                    None,
1843                    Some(format!(
1844                        "unparsable fix report after {attempts} continuation(s): {last_err}"
1845                    )),
1846                    ContinuationRecord {
1847                        attempts,
1848                        cumulative_wait_ms,
1849                        outcome: ContinuationOutcome::Exhausted,
1850                    },
1851                );
1852            }
1853            attempts += 1;
1854            self.state.event(
1855                "fix",
1856                format!(
1857                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
1858                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
1859                ),
1860            );
1861            let mut retry = job.clone();
1862            retry.seat = seat.clone();
1863            retry.prompt = prompt::resume_incomplete(&last_err);
1864            retry.timeout = retry_budget(job.timeout, true);
1865            retry.stem = format!("{}-continue{attempts}", job.stem);
1866            let cache = self.state.config.cache_dir();
1867            let ctx = WaveCtx {
1868                run: run_id,
1869                node: "fix",
1870                prompts,
1871                cache: cache.as_deref(),
1872                round: Some(round),
1873            };
1874            let (resumed_seat, resumed_out) = run_one(
1875                retry,
1876                Arc::clone(&self.sem),
1877                &ctx,
1878                &mut self.state,
1879                attempts,
1880            )
1881            .await;
1882            seat = resumed_seat;
1883            match resumed_out {
1884                AgentOutcome::Ok(o) => {
1885                    cumulative_wait_ms += o.duration_ms;
1886                    match verdict::extract_json::<FixReport>(&o.text) {
1887                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
1888                            self.state.event(
1889                                "fix",
1890                                format!(
1891                                    "round {round}: fixer's adoption report recovered after \
1892                                     {attempts} continuation(s)"
1893                                ),
1894                            );
1895                            return (
1896                                seat,
1897                                Some(report),
1898                                None,
1899                                ContinuationRecord {
1900                                    attempts,
1901                                    cumulative_wait_ms,
1902                                    outcome: ContinuationOutcome::Resumed,
1903                                },
1904                            );
1905                        }
1906                        // The report parsed, but this same reply's own
1907                        // CommandEvidence — the identical record `state.jobs`
1908                        // renders — names a command whose CLI never
1909                        // confirmed an exit status. Read together, that is
1910                        // not a resolved answer: keep nudging rather than
1911                        // accept a report standing next to a command the
1912                        // seat's own CLI cannot vouch for.
1913                        Ok(_) => {
1914                            last_err = "the reply parsed, but it reported a command whose own CLI \
1915                                 never confirmed an exit status"
1916                                .to_owned();
1917                        }
1918                        Err(e) => last_err = e.to_string(),
1919                    }
1920                }
1921                AgentOutcome::Quota(o) => {
1922                    cumulative_wait_ms += o.duration_ms;
1923                    self.state.quota.push(QuotaLoss {
1924                        seat: seat.key.clone(),
1925                        node: "fix".to_owned(),
1926                        at: Timestamp::now(),
1927                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1928                    });
1929                    self.state.event(
1930                        "fix",
1931                        format!(
1932                            "round {round}: continuation rate limited (quota); not retrying now"
1933                        ),
1934                    );
1935                    return (
1936                        seat,
1937                        None,
1938                        Some("rate limited (quota) while recovering the fix report".to_owned()),
1939                        ContinuationRecord {
1940                            attempts,
1941                            cumulative_wait_ms,
1942                            outcome: ContinuationOutcome::QuotaLost,
1943                        },
1944                    );
1945                }
1946                AgentOutcome::Dropped(o) => {
1947                    cumulative_wait_ms += o.duration_ms;
1948                    let why = o
1949                        .dropped
1950                        .as_ref()
1951                        .map(|d| d.why.as_str())
1952                        .unwrap_or("the CLI ended the stream without delivering its answer");
1953                    last_err = format!("the CLI dropped the stream ({why})");
1954                }
1955                AgentOutcome::Failed(e) => last_err = e,
1956            }
1957        }
1958    }
1959
1960    fn after_implement(&mut self) -> Result<()> {
1961        // Scan every candidate patch once the set is complete.
1962        if self.state.leaks.is_empty() {
1963            let cfg = self.state.config.blind.clone();
1964            let mut leaks = Vec::new();
1965            for c in &self.state.candidates {
1966                let Some(patch) =
1967                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
1968                else {
1969                    continue;
1970                };
1971                leaks.extend(blind::scan(
1972                    &format!("candidate {} patch", c.label),
1973                    &patch,
1974                    &cfg.vendor_tokens,
1975                ));
1976            }
1977            if !leaks.is_empty() {
1978                let summary = leaks
1979                    .iter()
1980                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
1981                    .collect::<Vec<_>>()
1982                    .join(", ");
1983                match cfg.on_leak {
1984                    LeakPolicy::Fail => {
1985                        self.state.status = RunStatus::Failed;
1986                        self.state
1987                            .event("blind", format!("vendor text in a patch: {summary}"));
1988                        self.state.leaks = leaks;
1989                        self.state.save()?;
1990                        self.settle_questions();
1991                        bail!(
1992                            "blind.on_leak = \"fail\" and vendor text reached a \
1993                             judged patch: {summary}"
1994                        );
1995                    }
1996                    LeakPolicy::Redact => self.state.event(
1997                        "blind",
1998                        format!("redacting vendor text for judging: {summary}"),
1999                    ),
2000                    LeakPolicy::Warn => self.state.event(
2001                        "blind",
2002                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
2003                    ),
2004                }
2005                self.state.leaks = leaks;
2006            }
2007        }
2008
2009        if self.state.viable().is_empty() {
2010            if self.state.all_candidates_verified_noop() {
2011                // Every candidate agreed, with evidence the adoption guard
2012                // accepted, that nothing belongs in this worktree. That is
2013                // not the same fact as a candidate that simply failed to
2014                // write anything, and settling it as an ordinary `Failed`
2015                // (see `SCHEMA`'s doc for schema 10) is what let two of
2016                // task 391f's attempts burn a retry each re-discovering the
2017                // same already-landed fix. Terminal either way, so `judge`
2018                // must never run over an empty candidate set — unlike the
2019                // `Failed` branch below this returns `Ok`, not an error:
2020                // nothing here failed.
2021                self.state.status = RunStatus::VerifiedNoop;
2022                self.state.save()?;
2023                self.settle_questions();
2024                return Ok(());
2025            }
2026            self.state.status = RunStatus::Failed;
2027            self.state.save()?;
2028            self.settle_questions();
2029            bail!("no candidate produced a change; nothing to judge");
2030        }
2031        self.state.status = RunStatus::Judging;
2032        self.state.save()?;
2033        Ok(())
2034    }
2035
2036    // --------------------------------------------------------------- judge
2037
2038    async fn judge(&mut self) -> Result<()> {
2039        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2040        // agent files with `magi task add` name the run that paid for it. The
2041        // prompt overlay is cloned alongside it because the waves borrow it
2042        // while `self` is mutably borrowed by the node's own bookkeeping.
2043        let run_id = self.state.id.clone();
2044        let prompts = self.state.config.prompts.clone();
2045        if !self.state.judgements.is_empty() || self.state.judge_skipped {
2046            return Ok(());
2047        }
2048        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2049        if viable.len() == 1 {
2050            // Recorded so this is a one-time event: `judgements` stays empty
2051            // either way, which without this flag is indistinguishable from
2052            // "not yet judged" on the next reentry — and status is left
2053            // untouched, so a later node's conclusion (e.g. `Blocked` after
2054            // the review budget ran out) survives a resume instead of being
2055            // clobbered back to `Judging` by this node running again.
2056            self.state.judge_skipped = true;
2057            self.state.event(
2058                "judge",
2059                format!(
2060                    "only candidate {} produced a change; judging skipped",
2061                    viable[0].label
2062                ),
2063            );
2064            self.state.save()?;
2065            return Ok(());
2066        }
2067        self.state.status = RunStatus::Judging;
2068
2069        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2070        let language = self.state.config.graph.language.clone();
2071        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2072        let sessions = self.state.config.graph.sessions;
2073        let artifacts = agent::artifacts_dir(&self.state.dir());
2074        let root = self.state.worktree_root();
2075        let base_short = short(&self.state.base_commit);
2076
2077        let mut jobs = Vec::new();
2078        let mut orders = Vec::new();
2079        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2080            let order = blind::presentation_order(viable.len(), j, self.state.seed);
2081            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2082            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
2083            let seat_key = format!("judge-{}", j + 1);
2084            let seat = self.seat(&seat_key, &spec.id);
2085            jobs.push(SeatJob {
2086                prompt: prompt::judge(
2087                    &self.state.instruction,
2088                    &views,
2089                    self.roles.judges.len(),
2090                    &base_short,
2091                    &language,
2092                ),
2093                spec,
2094                seat,
2095                cwd: root.join(format!("judge-{}", j + 1)),
2096                timeout,
2097                allow_write: false,
2098                sessions,
2099                artifacts: artifacts.clone(),
2100                stem: format!("judge-{}", j + 1),
2101            });
2102        }
2103
2104        self.state.event(
2105            "judge",
2106            format!(
2107                "{} judges ranking {} candidates blind",
2108                jobs.len(),
2109                viable.len()
2110            ),
2111        );
2112        let labels_for_check = labels.clone();
2113        let mut quota_losses = Vec::new();
2114        let cache = self.state.config.cache_dir();
2115        let ctx = WaveCtx {
2116            run: &run_id,
2117            node: "judge",
2118            prompts: &prompts,
2119            cache: cache.as_deref(),
2120            round: None,
2121        };
2122        let results = ask_json_wave::<Ranking>(
2123            jobs,
2124            Arc::clone(&self.sem),
2125            self.state.config.graph.retries,
2126            &ctx,
2127            &mut quota_losses,
2128            &mut self.state,
2129            &move |r: &Ranking| r.validate(&labels_for_check),
2130        )
2131        .await;
2132        self.state.quota.extend(quota_losses);
2133
2134        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
2135            let agent_id = seat.agent.clone();
2136            self.state.seats.insert(seat.key.clone(), seat);
2137            let mut record = Judgement {
2138                judge: j + 1,
2139                seat: format!("judge-{}", j + 1),
2140                agent: agent_id,
2141                ranking: Vec::new(),
2142                reasons: BTreeMap::new(),
2143                confidence: None,
2144                order: orders[j].clone(),
2145                failed: None,
2146                duration_ms: 0,
2147            };
2148            match res {
2149                Ok((ranking, out)) => {
2150                    record.ranking = ranking.normalized();
2151                    record.reasons = ranking.reasons;
2152                    record.confidence = ranking.confidence;
2153                    record.duration_ms = out.duration_ms;
2154                    self.state.event(
2155                        "judge",
2156                        format!(
2157                            "judge {} ranked {}",
2158                            j + 1,
2159                            record.ranking.iter().collect::<String>()
2160                        ),
2161                    );
2162                }
2163                Err(e) => {
2164                    record.failed = Some(e.to_string());
2165                    self.state
2166                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
2167                }
2168            }
2169            self.state.judgements.push(record);
2170            self.state.save()?;
2171        }
2172        Ok(())
2173    }
2174
2175    // ---------------------------------------------------------- deliberate
2176
2177    async fn deliberate(&mut self) -> Result<()> {
2178        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2179        // agent files with `magi task add` name the run that paid for it. The
2180        // prompt overlay is cloned alongside it because the waves borrow it
2181        // while `self` is mutably borrowed by the node's own bookkeeping.
2182        let run_id = self.state.id.clone();
2183        let prompts = self.state.config.prompts.clone();
2184        if !self.state.deliberation.is_empty() {
2185            return Ok(());
2186        }
2187        let tops: Vec<char> = self
2188            .state
2189            .judgements
2190            .iter()
2191            .filter_map(|j| j.ranking.first().copied())
2192            .collect();
2193        let rounds = self.state.config.graph.deliberate_rounds;
2194        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
2195            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
2196                self.state.event(
2197                    "deliberate",
2198                    format!("judges agreed on {} outright; no deliberation", tops[0]),
2199                );
2200            }
2201            self.state.status = RunStatus::Voting;
2202            self.state.save()?;
2203            return Ok(());
2204        }
2205
2206        self.state.status = RunStatus::Deliberating;
2207        self.state.event(
2208            "deliberate",
2209            format!(
2210                "split: first choices were {} — opening {rounds} round(s)",
2211                tops.iter().collect::<String>()
2212            ),
2213        );
2214
2215        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2216        let language = self.state.config.graph.language.clone();
2217        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2218        let sessions = self.state.config.graph.sessions;
2219        let artifacts = agent::artifacts_dir(&self.state.dir());
2220        let root = self.state.worktree_root();
2221        let base_short = short(&self.state.base_commit);
2222
2223        // Judges argue in sequence so that a turn can answer the one before it;
2224        // that is the difference between deliberation and three parallel
2225        // monologues.
2226        for round in 1..=rounds {
2227            let mut turns: Vec<DeliberationTurn> = Vec::new();
2228            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2229                if self.state.judgements[j].failed.is_some() {
2230                    continue;
2231                }
2232                let seat_key = format!("judge-{}", j + 1);
2233                let mut seat = self.seat(&seat_key, &spec.id);
2234                let transcript = self.transcript(&turns, j);
2235                let context = if has_context(&spec, &seat, sessions) {
2236                    None
2237                } else {
2238                    Some(self.candidate_block(&viable, &base_short))
2239                };
2240                let text = prompt::deliberate(
2241                    &self.state.instruction,
2242                    context.as_deref(),
2243                    &transcript,
2244                    round,
2245                    rounds,
2246                    &language,
2247                );
2248                let job = SeatJob {
2249                    spec,
2250                    seat: seat.clone(),
2251                    prompt: text,
2252                    cwd: root.join(format!("judge-{}", j + 1)),
2253                    timeout,
2254                    allow_write: false,
2255                    sessions,
2256                    artifacts: artifacts.clone(),
2257                    stem: format!("delib-{round}-judge-{}", j + 1),
2258                };
2259                let cache = self.state.config.cache_dir();
2260                let ctx = WaveCtx {
2261                    run: &run_id,
2262                    node: "deliberate",
2263                    prompts: &prompts,
2264                    cache: cache.as_deref(),
2265                    round: None,
2266                };
2267                let (updated, out) =
2268                    run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2269                seat = updated;
2270                let agent_id = seat.agent.clone();
2271                let seat_key = seat.key.clone();
2272                self.state.seats.insert(seat.key.clone(), seat);
2273                let body = match out {
2274                    AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
2275                    // Never read the CLI's raw error JSON as this judge's
2276                    // position — skip the seat instead, the same as any other
2277                    // failed turn.
2278                    AgentOutcome::Dropped(o) => {
2279                        let why =
2280                            o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
2281                                "the CLI ended the stream without delivering its answer",
2282                            );
2283                        self.state.event(
2284                            "deliberate",
2285                            format!(
2286                                "judge {} skipped: the CLI dropped the stream ({why})",
2287                                j + 1
2288                            ),
2289                        );
2290                        continue;
2291                    }
2292                    AgentOutcome::Quota(o) => {
2293                        self.state.quota.push(QuotaLoss {
2294                            seat: seat_key,
2295                            node: "deliberate".to_owned(),
2296                            at: Timestamp::now(),
2297                            reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2298                        });
2299                        self.state.event(
2300                            "deliberate",
2301                            format!("judge {} skipped: rate limited (quota)", j + 1),
2302                        );
2303                        continue;
2304                    }
2305                    AgentOutcome::Failed(e) => {
2306                        self.state
2307                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
2308                        continue;
2309                    }
2310                };
2311                let tentative = verdict::extract_json::<Position>(&body)
2312                    .ok()
2313                    .and_then(|p| p.tentative)
2314                    .and_then(|s| s.trim().chars().next())
2315                    .map(|c| c.to_ascii_uppercase());
2316                self.state.event(
2317                    "deliberate",
2318                    format!(
2319                        "round {round}: judge {} now favours {}",
2320                        j + 1,
2321                        tentative.map_or("—".to_owned(), |c| c.to_string())
2322                    ),
2323                );
2324                turns.push(DeliberationTurn {
2325                    judge: j + 1,
2326                    agent: agent_id,
2327                    body: blind::sanitize_prose(&body, &self.state.config.blind),
2328                    tentative,
2329                });
2330            }
2331            self.state
2332                .deliberation
2333                .push(DeliberationRound { round, turns });
2334            self.state.save()?;
2335        }
2336
2337        self.state.status = RunStatus::Voting;
2338        self.state.save()?;
2339        Ok(())
2340    }
2341
2342    // ---------------------------------------------------------------- vote
2343
2344    async fn vote(&mut self) -> Result<()> {
2345        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2346        // agent files with `magi task add` name the run that paid for it. The
2347        // prompt overlay is cloned alongside it because the waves borrow it
2348        // while `self` is mutably borrowed by the node's own bookkeeping.
2349        let run_id = self.state.id.clone();
2350        let prompts = self.state.config.prompts.clone();
2351        if !self.state.votes.is_empty() {
2352            return Ok(());
2353        }
2354        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2355        if viable.len() == 1 {
2356            return Ok(());
2357        }
2358        self.state.status = RunStatus::Voting;
2359
2360        let language = self.state.config.graph.language.clone();
2361        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2362        let sessions = self.state.config.graph.sessions;
2363        let artifacts = agent::artifacts_dir(&self.state.dir());
2364        let root = self.state.worktree_root();
2365        let base_short = short(&self.state.base_commit);
2366        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2367
2368        let mut jobs = Vec::new();
2369        let mut seats_at = Vec::new();
2370        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2371            if self
2372                .state
2373                .judgements
2374                .get(j)
2375                .is_some_and(|r| r.failed.is_some())
2376            {
2377                continue;
2378            }
2379            let seat_key = format!("judge-{}", j + 1);
2380            let seat = self.seat(&seat_key, &spec.id);
2381            let mut text = prompt::final_vote(&viable, &language);
2382            if !has_context(&spec, &seat, sessions) {
2383                text = format!(
2384                    "{}\n\n# Candidates\n\n{}",
2385                    text,
2386                    self.candidate_block(&candidates, &base_short)
2387                );
2388            }
2389            jobs.push(SeatJob {
2390                spec,
2391                seat,
2392                prompt: text,
2393                cwd: root.join(format!("judge-{}", j + 1)),
2394                timeout,
2395                allow_write: false,
2396                sessions,
2397                artifacts: artifacts.clone(),
2398                stem: format!("vote-judge-{}", j + 1),
2399            });
2400            seats_at.push(j);
2401        }
2402
2403        self.state.event(
2404            "vote",
2405            format!(
2406                "collecting {} final votes one by one, privately",
2407                jobs.len()
2408            ),
2409        );
2410        let allowed = viable.clone();
2411        let mut quota_losses = Vec::new();
2412        let cache = self.state.config.cache_dir();
2413        let ctx = WaveCtx {
2414            run: &run_id,
2415            node: "vote",
2416            prompts: &prompts,
2417            cache: cache.as_deref(),
2418            round: None,
2419        };
2420        let results = ask_json_wave::<FinalVote>(
2421            jobs,
2422            Arc::clone(&self.sem),
2423            self.state.config.graph.retries,
2424            &ctx,
2425            &mut quota_losses,
2426            &mut self.state,
2427            &move |v: &FinalVote| match v.label() {
2428                Some(c) if allowed.contains(&c) => Ok(()),
2429                other => bail!("vote {other:?} is not one of {allowed:?}"),
2430            },
2431        )
2432        .await;
2433        self.state.quota.extend(quota_losses);
2434
2435        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
2436            let agent_id = seat.agent.clone();
2437            self.state.seats.insert(seat.key.clone(), seat);
2438            let initial = self
2439                .state
2440                .judgements
2441                .get(j)
2442                .and_then(|r| r.ranking.first().copied());
2443            let mut record = VoteRecord {
2444                judge: j + 1,
2445                agent: agent_id,
2446                vote: None,
2447                reason: String::new(),
2448                changed: false,
2449            };
2450            match res {
2451                Ok((v, _)) => {
2452                    record.vote = v.label();
2453                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2454                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
2455                    self.state.event(
2456                        "vote",
2457                        format!(
2458                            "judge {} voted {}{}",
2459                            j + 1,
2460                            record.vote.unwrap_or('?'),
2461                            if record.changed { " (changed)" } else { "" }
2462                        ),
2463                    );
2464                }
2465                Err(e) => {
2466                    self.state
2467                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
2468                }
2469            }
2470            self.state.votes.push(record);
2471            self.state.save()?;
2472        }
2473        Ok(())
2474    }
2475
2476    // --------------------------------------------------------------- tally
2477
2478    fn tally(&mut self) -> Result<()> {
2479        if self.state.tally.is_some() {
2480            return Ok(());
2481        }
2482        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2483        let tops: Vec<char> = self
2484            .state
2485            .judgements
2486            .iter()
2487            .filter_map(|j| j.ranking.first().copied())
2488            .collect();
2489        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
2490
2491        // A judge whose private vote failed still counted once, in the initial
2492        // ranking; using it beats discarding a whole seat.
2493        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2494        let mut cast: Vec<char> = Vec::new();
2495        for (i, j) in self.state.judgements.iter().enumerate() {
2496            let vote = self
2497                .state
2498                .votes
2499                .iter()
2500                .find(|v| v.judge == i + 1)
2501                .and_then(|v| v.vote)
2502                .or_else(|| j.ranking.first().copied());
2503            if let Some(v) = vote {
2504                *first_choice.entry(v).or_insert(0) += 1;
2505                cast.push(v);
2506            }
2507        }
2508
2509        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2510        for j in &self.state.judgements {
2511            let n = j.ranking.len();
2512            for (pos, label) in j.ranking.iter().enumerate() {
2513                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
2514            }
2515        }
2516
2517        let best = first_choice.values().copied().max().unwrap_or(0);
2518        let mut leaders: Vec<char> = first_choice
2519            .iter()
2520            .filter(|(_, v)| **v == best)
2521            .map(|(k, _)| *k)
2522            .collect();
2523        let mut tie_break = None;
2524        if leaders.len() > 1 {
2525            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
2526            let borda_leaders: Vec<char> = leaders
2527                .iter()
2528                .copied()
2529                .filter(|l| borda[l] == top_borda)
2530                .collect();
2531            tie_break = Some(if borda_leaders.len() == 1 {
2532                format!(
2533                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
2534                    leaders.len()
2535                )
2536            } else {
2537                format!(
2538                    "{} way tie on both first-choice votes and Borda points, broken by label order",
2539                    leaders.len()
2540                )
2541            });
2542            leaders = borda_leaders;
2543            leaders.sort_unstable();
2544        }
2545        let winner = *leaders
2546            .first()
2547            .or(viable.first())
2548            .context("no candidate to declare a winner from")?;
2549
2550        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
2551        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
2552        let deliberated = !self.state.deliberation.is_empty();
2553
2554        // Whose verdict is this? A rate-limited seat is absent even if it
2555        // ranked before the limit hit, so presence is measured against the
2556        // recorded losses, not just "did a ranking ever appear".
2557        let quota_seats: std::collections::BTreeSet<&str> =
2558            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2559        let mut present = 0usize;
2560        for (i, j) in self.state.judgements.iter().enumerate() {
2561            if quota_seats.contains(j.seat.as_str()) {
2562                continue;
2563            }
2564            let ranked = !j.ranking.is_empty() && j.failed.is_none();
2565            let voted = self
2566                .state
2567                .votes
2568                .iter()
2569                .any(|v| v.judge == i + 1 && v.vote.is_some());
2570            if ranked || voted {
2571                present += 1;
2572            }
2573        }
2574        // Strict majority of the configured panel. A bare majority is real
2575        // signal we can act on, while a minority verdict must never stand in
2576        // for a healthy one. A one-candidate run needs no panel at all, and
2577        // `judges` stays `0` rather than the roster size a panel that never
2578        // sat would otherwise be credited with.
2579        let needs_quorum = viable.len() > 1;
2580        let judges_total = if needs_quorum {
2581            self.roles.judges.len()
2582        } else {
2583            0
2584        };
2585        let quorum = if needs_quorum {
2586            judges_total / 2 + 1
2587        } else {
2588            0
2589        };
2590        let met_quorum = !needs_quorum || present >= quorum;
2591        let uncontested = (!needs_quorum).then(|| {
2592            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
2593        });
2594
2595        self.state.event(
2596            "tally",
2597            match &uncontested {
2598                Some(reason) => format!("winner {winner} — {reason}"),
2599                None => format!(
2600                    "winner {winner} — votes {} | initial {} | {} changed | \
2601                     {present}/{judges_total} judges{}",
2602                    first_choice
2603                        .iter()
2604                        .map(|(k, v)| format!("{k}:{v}"))
2605                        .collect::<Vec<_>>()
2606                        .join(" "),
2607                    if unanimous_initial {
2608                        "unanimous"
2609                    } else {
2610                        "split"
2611                    },
2612                    changed_votes,
2613                    if met_quorum {
2614                        String::new()
2615                    } else {
2616                        format!(" — below quorum ({quorum} required)")
2617                    },
2618                ),
2619            },
2620        );
2621        if !met_quorum {
2622            self.state.event(
2623                "stall",
2624                format!(
2625                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
2626                     the run stops here, resumable"
2627                ),
2628            );
2629        }
2630        self.state.tally = Some(Tally {
2631            first_choice,
2632            borda,
2633            winner,
2634            rankings: tops.len(),
2635            unanimous_initial,
2636            deliberated,
2637            changed_votes,
2638            unanimous_final,
2639            tie_break,
2640            judges: judges_total,
2641            present,
2642            quorum,
2643            met_quorum,
2644            uncontested,
2645        });
2646        self.state.status = if met_quorum {
2647            RunStatus::Reviewing
2648        } else {
2649            RunStatus::Stalled
2650        };
2651        self.state.save()?;
2652        Ok(())
2653    }
2654
2655    // ------------------------------------------------------------- recover
2656
2657    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
2658    /// resumed toward completion once the transient cause clears.
2659    ///
2660    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
2661    /// it toward the quorum, which is exactly the set of seats whose absence
2662    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
2663    /// not depend on which node happened to hit the limit), or an ordinary
2664    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
2665    /// seat is never disturbed.
2666    ///
2667    /// A seat that now answers with a usable ranking is "recovered": its
2668    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
2669    /// `tally` counts it present again), and its vote re-collected. A seat that
2670    /// still fails keeps its loss and stays absent.
2671    ///
2672    /// Returns `true` when the re-tally restores the quorum (the run may proceed
2673    /// to review/gate/merge), `false` when it is still below quorum (the run
2674    /// stays `Stalled`, still resumable for a later retry).
2675    #[allow(clippy::too_many_lines)]
2676    async fn recover_stall(&mut self) -> Result<bool> {
2677        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2678        // agent files with `magi task add` name the run that paid for it. The
2679        // prompt overlay is cloned alongside it because the waves borrow it
2680        // while `self` is mutably borrowed by the node's own bookkeeping.
2681        let run_id = self.state.id.clone();
2682        let prompts = self.state.config.prompts.clone();
2683        // Absent seats = quota-lost at any node, or failed outright. Mirroring
2684        // `tally`'s presence test (rather than the old quota-judge/vote filter)
2685        // is what keeps a non-quota collapse — or a quota loss recorded at the
2686        // deliberate node — from being a permanent dead-end on `--resume`.
2687        let quota_seats: BTreeSet<&str> =
2688            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2689        let absent: Vec<String> = self
2690            .state
2691            .judgements
2692            .iter()
2693            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
2694            .map(|j| j.seat.clone())
2695            .collect();
2696        if absent.is_empty() {
2697            return Ok(false);
2698        }
2699        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2700        if viable.len() <= 1 {
2701            return Ok(false);
2702        }
2703        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2704        let language = self.state.config.graph.language.clone();
2705        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2706        let sessions = self.state.config.graph.sessions;
2707        let artifacts = agent::artifacts_dir(&self.state.dir());
2708        let root = self.state.worktree_root();
2709        let base_short = short(&self.state.base_commit);
2710        let candidates: Vec<Candidate> = viable.clone();
2711
2712        // Map each absent seat key to its 0-based position in `roles.judges`.
2713        let mut positions: Vec<usize> = absent
2714            .iter()
2715            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
2716            .collect();
2717        if positions.is_empty() {
2718            return Ok(false);
2719        }
2720        positions.sort_unstable();
2721        positions.dedup();
2722
2723        // Re-rank the lost seats, one blind prompt each.
2724        let mut judge_jobs = Vec::new();
2725        for &j in &positions {
2726            let order = blind::presentation_order(viable.len(), j, self.state.seed);
2727            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2728            let seat_key = format!("judge-{}", j + 1);
2729            let spec = self.roles.judges[j].clone();
2730            let seat = self.seat(&seat_key, &spec.id);
2731            judge_jobs.push(SeatJob {
2732                spec,
2733                seat,
2734                prompt: prompt::judge(
2735                    &self.state.instruction,
2736                    &views,
2737                    self.roles.judges.len(),
2738                    &base_short,
2739                    &language,
2740                ),
2741                cwd: root.join(seat_key),
2742                timeout,
2743                allow_write: false,
2744                sessions,
2745                artifacts: artifacts.clone(),
2746                stem: format!("judge-{}-recover", j + 1),
2747            });
2748        }
2749
2750        let labels_for_check = labels.clone();
2751        let mut judge_losses = Vec::new();
2752        let retries = self.state.config.graph.retries;
2753        let cache = self.state.config.cache_dir();
2754        let ctx = WaveCtx {
2755            run: &run_id,
2756            node: "judge",
2757            prompts: &prompts,
2758            cache: cache.as_deref(),
2759            round: None,
2760        };
2761        let results = ask_json_wave::<Ranking>(
2762            judge_jobs,
2763            Arc::clone(&self.sem),
2764            retries,
2765            &ctx,
2766            &mut judge_losses,
2767            &mut self.state,
2768            &move |r: &Ranking| r.validate(&labels_for_check),
2769        )
2770        .await;
2771
2772        // Refresh the judgement of every seat that ranked again.
2773        let mut recovered: BTreeSet<usize> = BTreeSet::new();
2774        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
2775            self.state.seats.insert(seat.key.clone(), seat);
2776            let record = &mut self.state.judgements[j];
2777            match res {
2778                Ok((ranking, out)) => {
2779                    record.ranking = ranking.normalized();
2780                    record.reasons = ranking.reasons;
2781                    record.confidence = ranking.confidence;
2782                    record.failed = None;
2783                    record.duration_ms = out.duration_ms;
2784                    recovered.insert(j);
2785                    self.state.event(
2786                        "recover",
2787                        format!("judge {} ranked again after the limit", j + 1),
2788                    );
2789                }
2790                Err(e) => {
2791                    self.state
2792                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
2793                }
2794            }
2795        }
2796
2797        // Re-ask the votes of the seats that recovered a ranking.
2798        let mut vote_jobs = Vec::new();
2799        let mut vote_pos: Vec<usize> = Vec::new();
2800        for &j in &recovered {
2801            let seat_key = format!("judge-{}", j + 1);
2802            let spec = self.roles.judges[j].clone();
2803            let seat = self.seat(&seat_key, &spec.id);
2804            let mut text = prompt::final_vote(&labels, &language);
2805            if !has_context(&spec, &seat, sessions) {
2806                text = format!(
2807                    "{}\n\n# Candidates\n\n{}",
2808                    text,
2809                    self.candidate_block(&candidates, &base_short)
2810                );
2811            }
2812            vote_jobs.push(SeatJob {
2813                spec,
2814                seat,
2815                prompt: text,
2816                cwd: root.join(seat_key),
2817                timeout,
2818                allow_write: false,
2819                sessions,
2820                artifacts: artifacts.clone(),
2821                stem: format!("vote-judge-{}-recover", j + 1),
2822            });
2823            vote_pos.push(j);
2824        }
2825        let allowed = labels.clone();
2826        let mut vote_losses = Vec::new();
2827        let vote_retries = self.state.config.graph.retries;
2828        let vote_cache = self.state.config.cache_dir();
2829        let ctx = WaveCtx {
2830            run: &run_id,
2831            node: "vote",
2832            prompts: &prompts,
2833            cache: vote_cache.as_deref(),
2834            round: None,
2835        };
2836        let votes = ask_json_wave::<FinalVote>(
2837            vote_jobs,
2838            Arc::clone(&self.sem),
2839            vote_retries,
2840            &ctx,
2841            &mut vote_losses,
2842            &mut self.state,
2843            &move |v: &FinalVote| match v.label() {
2844                Some(c) if allowed.contains(&c) => Ok(()),
2845                other => bail!("vote {other:?} is not one of {allowed:?}"),
2846            },
2847        )
2848        .await;
2849        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
2850            let agent_id = seat.agent.clone();
2851            self.state.seats.insert(seat.key.clone(), seat);
2852            match res {
2853                Ok((v, _)) => {
2854                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
2855                        rec.vote = v.label();
2856                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2857                    } else {
2858                        self.state.votes.push(VoteRecord {
2859                            judge: j + 1,
2860                            agent: agent_id,
2861                            vote: v.label(),
2862                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
2863                            changed: false,
2864                        });
2865                    }
2866                    self.state.event(
2867                        "recover",
2868                        format!("judge {} voted again after the limit", j + 1),
2869                    );
2870                }
2871                Err(e) => {
2872                    self.state
2873                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
2874                }
2875            }
2876        }
2877
2878        // A seat that ranked again is present even if its re-vote failed —
2879        // `tally` falls back to the initial ranking's first choice — so clear
2880        // its quota loss. Seats that still fail keep theirs and stay absent.
2881        let recovered_keys: BTreeSet<String> = recovered
2882            .iter()
2883            .map(|&j| format!("judge-{}", j + 1))
2884            .collect();
2885        self.state
2886            .quota
2887            .retain(|q| !recovered_keys.contains(&q.seat));
2888        // A seat that hit the limit again is a fresh loss, not the old one:
2889        // replace the stale entry so the history stays one-per-seat and the
2890        // daemon can tell this attempt's loss from a previous session's.
2891        for loss in judge_losses.into_iter().chain(vote_losses) {
2892            if recovered_keys.contains(&loss.seat) {
2893                continue;
2894            }
2895            self.state.quota.retain(|q| q.seat != loss.seat);
2896            self.state.quota.push(loss);
2897        }
2898
2899        // Recompute the verdict from the refreshed panel.
2900        self.state.tally = None;
2901        self.tally()?;
2902        Ok(self
2903            .state
2904            .tally
2905            .as_ref()
2906            .map(|t| t.met_quorum)
2907            .unwrap_or(false))
2908    }
2909
2910    // ----------------------------------------------------------------- fold
2911
2912    async fn fold_losers(&mut self) -> Result<()> {
2913        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
2914            return Ok(());
2915        };
2916        let repo = self.state.repo.clone();
2917        let mut folded = Vec::new();
2918        for i in 0..self.state.candidates.len() {
2919            let c = &self.state.candidates[i];
2920            if c.label == winner || c.folded {
2921                continue;
2922            }
2923            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
2924            git::worktree_remove(&repo, &wt).await.ok();
2925            git::branch_delete(&repo, &branch).await.ok();
2926            self.state.candidates[i].folded = true;
2927            folded.push(label.to_string());
2928        }
2929        // The judges are finished; their checkouts are pure cost from here.
2930        let root = self.state.worktree_root();
2931        for j in 1..=self.roles.judges.len() {
2932            let wt = root.join(format!("judge-{j}"));
2933            if wt.exists() {
2934                git::worktree_remove(&repo, &wt).await.ok();
2935            }
2936        }
2937        // The design-deliberation stage is finished by the time a tally
2938        // exists — same reasoning as the judges above.
2939        if self.state.config.graph.advise {
2940            for k in 1..=self.state.config.graph.advisors {
2941                let wt = root.join(format!("advisor-{k}"));
2942                if wt.exists() {
2943                    git::worktree_remove(&repo, &wt).await.ok();
2944                }
2945            }
2946        }
2947        if !folded.is_empty() {
2948            self.state
2949                .event("fold", format!("folded candidates {}", folded.join(", ")));
2950            self.state.save()?;
2951        }
2952        Ok(())
2953    }
2954
2955    // ------------------------------------------------------------ base sync
2956
2957    /// Land the winner's tree on the current tip of `<remote>/<base>` before
2958    /// anything verifies it.
2959    ///
2960    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
2961    /// read whatever is checked out in the winner's worktree. Left alone that
2962    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
2963    /// it when the run *branched* - and a run takes long enough that the base
2964    /// has usually moved by the time it gets here. A gate that ran there
2965    /// answers "green on the commit this run started from", not "green on
2966    /// what is about to land", and the difference showed up three times in
2967    /// one day as a green run whose merge would have reverted a file another
2968    /// pull request had already landed.
2969    ///
2970    /// Reuses [`git::rebase_branch_in_temp`] rather than a second
2971    /// implementation of the same idea: `land::Step::Rebase` already worked
2972    /// out the rules - throwaway worktree, conflict stops and reports rather
2973    /// than feeding a fixer, nothing runs in the primary tree - and a second
2974    /// rebase path is exactly the kind of drift `resolve_base`'s own doc
2975    /// warns about ("two answers to a question nobody notices until a diff is
2976    /// wrong").
2977    ///
2978    /// Bounded by [`BASE_SYNC_ROUNDS`], counted in `state.base_sync.attempts`
2979    /// so it survives a park/resume. A conflict or a push failure sets
2980    /// `state.base_sync.conflict` and leaves the branch and worktree exactly
2981    /// as they were - untouched, for a person to look at - which is also what
2982    /// makes re-entering this function afterwards a no-op instead of a second
2983    /// attempt at the same wall.
2984    async fn sync_to_base(&mut self) -> Result<()> {
2985        if self
2986            .state
2987            .base_sync
2988            .as_ref()
2989            .is_some_and(|s| s.conflict.is_some())
2990        {
2991            return Ok(());
2992        }
2993        let Some(winner) = self.state.winner().cloned() else {
2994            return Ok(());
2995        };
2996
2997        let repo = self.state.repo.clone();
2998        let remote = self.state.config.merge.remote.clone();
2999        let base_branch = self.state.base_branch.clone();
3000        let tracking = format!("{remote}/{base_branch}");
3001
3002        git::fetch(&repo, &remote, &base_branch).await.ok();
3003        // No network, or the remote never had this branch: `resolve_base`
3004        // already treats that as non-fatal at branch time, and a run that got
3005        // this far must not be blocked by it here either.
3006        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
3007            return Ok(());
3008        };
3009
3010        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3011        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
3012        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
3013
3014        if behind == 0 {
3015            self.state.base_sync = Some(BaseSync {
3016                tip,
3017                behind: 0,
3018                attempts,
3019                conflict: None,
3020            });
3021            self.state.save()?;
3022            return Ok(());
3023        }
3024
3025        if attempts >= BASE_SYNC_ROUNDS {
3026            let why = format!(
3027                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
3028                 rebase(s); rebasing again would only race it",
3029                winner.branch
3030            );
3031            self.state.status = RunStatus::Blocked;
3032            self.state.base_sync = Some(BaseSync {
3033                tip,
3034                behind,
3035                attempts,
3036                conflict: Some(why.clone()),
3037            });
3038            self.state.event("land", why);
3039            self.state.save()?;
3040            return Ok(());
3041        }
3042
3043        self.state.event(
3044            "land",
3045            format!(
3046                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
3047                winner.branch
3048            ),
3049        );
3050        self.state.save()?;
3051
3052        let scratch = self.state.dir().join("base-sync");
3053        let rebased = git::rebase_branch_in_temp(&repo, &scratch, &winner.branch, &tracking).await;
3054        let attempts = attempts + 1;
3055        match rebased {
3056            Ok(None) => {
3057                // The branch ref moved, but a worktree that already had it
3058                // checked out (the winner's) was not told; sync its index and
3059                // files before anything reads them.
3060                git::sync_to_head(&winner.worktree).await?;
3061                self.state.base_sync = Some(BaseSync {
3062                    tip: tip.clone(),
3063                    behind: 0,
3064                    attempts,
3065                    conflict: None,
3066                });
3067                self.state
3068                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
3069            }
3070            Ok(Some(conflict)) => {
3071                let why = format!(
3072                    "{} conflicts with {tracking} and did not rebase: {}",
3073                    winner.branch,
3074                    conflict.chars().take(600).collect::<String>()
3075                );
3076                self.state.status = RunStatus::Blocked;
3077                self.state.base_sync = Some(BaseSync {
3078                    tip,
3079                    behind,
3080                    attempts,
3081                    conflict: Some(why.clone()),
3082                });
3083                self.state.event("land", why);
3084            }
3085            Err(e) => {
3086                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
3087                self.state.status = RunStatus::Blocked;
3088                self.state.base_sync = Some(BaseSync {
3089                    tip,
3090                    behind,
3091                    attempts,
3092                    conflict: Some(why.clone()),
3093                });
3094                self.state.event("land", why);
3095            }
3096        }
3097        self.state.save()?;
3098        Ok(())
3099    }
3100
3101    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
3102    /// last landed the winner on, once it has run, else the commit the run
3103    /// branched from.
3104    ///
3105    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
3106    /// and `vote` all happen before there is a winner to rebase, so they
3107    /// compare every candidate against the branch point on purpose, and a
3108    /// base that moves after they are already done cannot change an answer
3109    /// they already gave.
3110    fn landing_base(&self) -> String {
3111        self.state
3112            .base_sync
3113            .as_ref()
3114            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
3115    }
3116
3117    // ------------------------------------------------------- operator fix
3118
3119    /// Route specific, already-recorded review findings to a fixer for a
3120    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
3121    /// entry point.
3122    ///
3123    /// Distinct from `review_loop`'s own fix step in three ways: it never
3124    /// runs a reviewer wave, it never spends review-round budget, and what
3125    /// happened is recorded as an [`OperatorFixRequest`] appended to
3126    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
3127    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
3128    /// and vote must never be rewritten to look like a manufactured blocking
3129    /// verdict.
3130    ///
3131    /// Only meaningful once review has actually concluded: `Ready` (handed
3132    /// off with findings still open, or simply concluded clean while minor
3133    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
3134    /// gate failed). Everything else is refused: a run still in progress
3135    /// should simply be resumed, and a `Merged` run's branch has already
3136    /// landed — reopening *this* run's own record cannot change that, so the
3137    /// answer there is a fresh `magi review <branch>`.
3138    ///
3139    /// A real commit here re-verifies through a fresh, ordinary review-only
3140    /// run on the same branch ([`Self::review`]) rather than reopening this
3141    /// run's own `review_loop`: once any round in this run's history went
3142    /// clean, `review_conclusion` treats that as permanent by design (the
3143    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
3144    /// way to force one more genuine reviewer wave out of *this* run without
3145    /// either rewriting history or weakening that guarantee for every other
3146    /// caller. A review-only run costs nothing extra — no implementation, no
3147    /// judging, no vote — and exercises the exact same review → verify →
3148    /// gate → (human) merge path, unmodified.
3149    pub async fn fix_selected(
3150        &mut self,
3151        ids: &[String],
3152        reason: &str,
3153        allow_stale: bool,
3154    ) -> Result<()> {
3155        let reason = reason.trim();
3156        if reason.is_empty() {
3157            bail!("a fix request needs a reason — that is the operator's own record of why");
3158        }
3159        if ids.is_empty() {
3160            bail!("no finding id given");
3161        }
3162        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
3163            bail!(
3164                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
3165                 has already concluded — can be given a targeted fix. A run still \
3166                 in progress should simply be resumed; a `merged` run's branch has \
3167                 already landed, so its answer is a fresh `magi review <branch>`, \
3168                 not reopening this run's own record",
3169                self.state.id,
3170                self.state.status.as_str()
3171            );
3172        }
3173        let Some(winner) = self.state.winner().cloned() else {
3174            bail!("run {} has no winning candidate to fix", self.state.id);
3175        };
3176        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
3177            bail!(
3178                "branch `{}` no longer exists; this run cannot be extended",
3179                winner.branch
3180            );
3181        }
3182        let home = crate::run::home();
3183        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
3184            bail!(
3185                "run {} is currently being worked on by another magi process",
3186                self.state.id
3187            );
3188        }
3189        // Held for the rest of this call, including the follow-up review
3190        // below: two `magi fix` invocations against the same run must not
3191        // both reach the worktree manipulation further down, which would
3192        // otherwise race to remove and recreate the same directory — see
3193        // [`FixClaim`]'s own doc.
3194        let _claim = FixClaim::acquire(&self.state.dir())?;
3195
3196        // Resolve every id before spending anything — an unknown id refuses
3197        // the whole request rather than silently dropping it — and dedup
3198        // while keeping the operator's own order.
3199        let mut seen = BTreeSet::new();
3200        let mut findings = Vec::new();
3201        let mut missing = Vec::new();
3202        for id in ids {
3203            if !seen.insert(id.clone()) {
3204                continue;
3205            }
3206            match self.state.finding(id) {
3207                Some((round, rec, f)) => findings.push(OperatorFixFinding {
3208                    id: f.id.clone(),
3209                    severity: f.severity,
3210                    reviewer_vote: rec.vote,
3211                    round: round.round,
3212                    round_head: round.head.clone(),
3213                    reviewer: rec.reviewer,
3214                    agent: rec.agent.clone(),
3215                    file: f.file.clone(),
3216                    line: f.line,
3217                    title: f.title.clone(),
3218                    detail: f.detail.clone(),
3219                    outcome: OperatorFixOutcome::Pending,
3220                }),
3221                None => missing.push(id.clone()),
3222            }
3223        }
3224        if !missing.is_empty() {
3225            bail!(
3226                "unknown finding id(s): {}; nothing was changed",
3227                missing.join(", ")
3228            );
3229        }
3230
3231        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
3232        let stale_details: Vec<(String, String)> = findings
3233            .iter()
3234            .filter(|f| f.round_head != head_at_request)
3235            .map(|f| (f.id.clone(), f.round_head.clone()))
3236            .collect();
3237        let stale = !stale_details.is_empty();
3238        if stale && !allow_stale {
3239            bail!(
3240                "the branch has moved since some finding(s) were raised — {} — now \
3241                 at {}; pass --allow-stale to fix anyway, or re-run review first",
3242                stale_details
3243                    .iter()
3244                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
3245                    .collect::<Vec<_>>()
3246                    .join(", "),
3247                short(&head_at_request)
3248            );
3249        }
3250
3251        let request = OperatorFixRequest {
3252            requested_at: Timestamp::now(),
3253            reason: reason.to_owned(),
3254            findings,
3255            head_at_request: head_at_request.clone(),
3256            allow_stale,
3257            stale,
3258            fix: None,
3259            result_head: None,
3260            follow_up_review_run: None,
3261        };
3262        self.state.event(
3263            "fix",
3264            format!(
3265                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
3266                request.findings.len(),
3267                request
3268                    .findings
3269                    .iter()
3270                    .map(|f| f.id.as_str())
3271                    .collect::<Vec<_>>()
3272                    .join(", "),
3273            ),
3274        );
3275        // Recorded now, before any worktree work or the fixer call itself —
3276        // and re-saved at each checkpoint below: a crash at any point after
3277        // this (mid fixer call, mid follow-up review) must not lose the fact
3278        // that this was requested, for which findings, and why. Everything
3279        // past this point reads and writes through `request_index` rather
3280        // than a local variable, since `request` itself is moved here.
3281        self.state.operator_fixes.push(request);
3282        self.state.save()?;
3283        let request_index = self.state.operator_fixes.len() - 1;
3284
3285        // A fresh, dedicated worktree for this one call, never the winner's
3286        // own worktree in place: that one may already be gone (folded away),
3287        // and reusing it in place would leave the branch checked out there
3288        // when the follow-up review below tries to check it out again. Freed
3289        // immediately after, either way — but only once confirmed clean:
3290        // `worktree_remove` is a `git worktree remove --force`, which would
3291        // otherwise discard uncommitted work left there by the operator or
3292        // another process before this had a chance to even look at it.
3293        if winner.worktree.exists() {
3294            if !git::is_clean(&winner.worktree).await? {
3295                bail!(
3296                    "`{}` has uncommitted changes; refusing to touch it — commit or \
3297                     discard them first",
3298                    winner.worktree.display()
3299                );
3300            }
3301            git::worktree_remove(&self.state.repo, &winner.worktree)
3302                .await
3303                .ok();
3304        }
3305        let fix_worktree = self.state.worktree_root().join("operator-fix");
3306        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
3307        git::git(
3308            &self.state.repo,
3309            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
3310        )
3311        .await
3312        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
3313        if !git::is_clean(&fix_worktree).await? {
3314            git::worktree_remove(&self.state.repo, &fix_worktree)
3315                .await
3316                .ok();
3317            bail!(
3318                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
3319                winner.branch
3320            );
3321        }
3322
3323        let run_id = self.state.id.clone();
3324        let prompts = self.state.config.prompts.clone();
3325        let language = self.state.config.graph.language.clone();
3326        let sessions = self.state.config.graph.sessions;
3327        let artifacts = agent::artifacts_dir(&self.state.dir());
3328        let (fix_spec, fix_seat_key) = match &self.roles.fixer {
3329            Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3330            _ => (
3331                self.state
3332                    .config
3333                    .agent(&winner.agent)
3334                    .cloned()
3335                    .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3336                format!("impl-{}", winner.label),
3337            ),
3338        };
3339        let seat = self.seat(&fix_seat_key, &fix_spec.id);
3340        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
3341            .findings
3342            .iter()
3343            .map(|f| Finding {
3344                id: f.id.clone(),
3345                severity: f.severity,
3346                file: f.file.clone(),
3347                line: f.line,
3348                title: f.title.clone(),
3349                detail: f.detail.clone(),
3350            })
3351            .collect();
3352        let job = SeatJob {
3353            prompt: prompt::operator_fix(
3354                &self.state.instruction,
3355                &finding_list,
3356                reason,
3357                &stale_details,
3358                &head_at_request,
3359                &language,
3360            ),
3361            spec: fix_spec.clone(),
3362            seat,
3363            cwd: fix_worktree.clone(),
3364            timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
3365            allow_write: true,
3366            sessions,
3367            artifacts: artifacts.clone(),
3368            stem: "operator-fix".to_owned(),
3369        };
3370        let cache = self.state.config.cache_dir();
3371        let ctx = WaveCtx {
3372            run: &run_id,
3373            node: "fix",
3374            prompts: &prompts,
3375            cache: cache.as_deref(),
3376            round: None,
3377        };
3378        let (seat, out) =
3379            run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3380        let agent_id = seat.agent.clone();
3381
3382        let mut fix = FixRecord {
3383            agent: agent_id,
3384            addressed: Vec::new(),
3385            rejected: Vec::new(),
3386            notes: String::new(),
3387            committed: false,
3388            failed: None,
3389            duration_ms: 0,
3390            continuation: None,
3391        };
3392        let mut final_seat = seat.clone();
3393        match out {
3394            AgentOutcome::Ok(o) => {
3395                fix.duration_ms = o.duration_ms;
3396                let parsed = verdict::extract_json::<FixReport>(&o.text);
3397                let incomplete_reason = match &parsed {
3398                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
3399                        "the reply parsed, but it reported a command whose own CLI \
3400                         never confirmed an exit status"
3401                            .to_owned(),
3402                    ),
3403                    Ok(_) => None,
3404                    Err(e) => Some(e.to_string()),
3405                };
3406                match incomplete_reason {
3407                    None => {
3408                        let report = parsed.expect("checked Ok above");
3409                        fix.addressed = report.addressed;
3410                        fix.rejected = report.rejected;
3411                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
3412                    }
3413                    Some(reason) => {
3414                        let (resumed_seat, resolved, failure, cont) = self
3415                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
3416                            .await;
3417                        fix.duration_ms += cont.cumulative_wait_ms;
3418                        fix.continuation = Some(cont);
3419                        final_seat = resumed_seat;
3420                        match resolved {
3421                            Some(report) => {
3422                                fix.addressed = report.addressed;
3423                                fix.rejected = report.rejected;
3424                                fix.notes =
3425                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
3426                            }
3427                            None => fix.failed = failure,
3428                        }
3429                    }
3430                }
3431            }
3432            AgentOutcome::Dropped(o) => {
3433                fix.duration_ms = o.duration_ms;
3434                let why = o
3435                    .dropped
3436                    .as_ref()
3437                    .map(|d| d.why.as_str())
3438                    .unwrap_or("the CLI ended the stream without delivering its answer");
3439                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
3440            }
3441            AgentOutcome::Quota(o) => {
3442                self.state.quota.push(QuotaLoss {
3443                    seat: final_seat.key.clone(),
3444                    node: "fix".to_owned(),
3445                    at: Timestamp::now(),
3446                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3447                });
3448                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
3449            }
3450            AgentOutcome::Failed(e) => fix.failed = Some(e),
3451        }
3452        if fix.continuation.is_none() {
3453            fix.continuation = Some(ContinuationRecord::not_needed());
3454        }
3455        self.state.seats.insert(final_seat.key.clone(), final_seat);
3456
3457        git::commit_all(
3458            &fix_worktree,
3459            &format!(
3460                "magi: operator-selected fix ({}) (uncommitted work)",
3461                self.state.operator_fixes[request_index]
3462                    .findings
3463                    .iter()
3464                    .map(|f| f.id.as_str())
3465                    .collect::<Vec<_>>()
3466                    .join(", ")
3467            ),
3468        )
3469        .await
3470        .ok();
3471        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
3472        fix.committed = after != head_at_request;
3473        git::worktree_remove(&self.state.repo, &fix_worktree)
3474            .await
3475            .ok();
3476
3477        self.state.event(
3478            "fix",
3479            match &fix.failed {
3480                Some(reason) => format!(
3481                    "operator fix: adoption report was lost ({reason}); {}",
3482                    if fix.committed {
3483                        "committed"
3484                    } else {
3485                        "NO new commit"
3486                    }
3487                ),
3488                None => format!(
3489                    "operator fix: {} addressed, {} rejected, {}",
3490                    fix.addressed.len(),
3491                    fix.rejected.len(),
3492                    if fix.committed {
3493                        "committed"
3494                    } else {
3495                        "NO new commit"
3496                    }
3497                ),
3498            },
3499        );
3500
3501        // Every selected finding gets an outcome — never left `Pending` once
3502        // the fixer's own turn is over. A report that never came back at all
3503        // marks every one of them `Unreported`, not silently "not addressed":
3504        // quota, a dropped stream, or an exhausted continuation are gaps in
3505        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
3506        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
3507        for f in &mut self.state.operator_fixes[request_index].findings {
3508            f.outcome = if fix.failed.is_some() {
3509                OperatorFixOutcome::Unreported
3510            } else if fix.addressed.contains(&f.id) {
3511                OperatorFixOutcome::Addressed
3512            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
3513                OperatorFixOutcome::Rejected { why: r.why.clone() }
3514            } else {
3515                OperatorFixOutcome::Unreported
3516            };
3517        }
3518
3519        let committed = fix.committed;
3520        if committed {
3521            self.state.operator_fixes[request_index].result_head = Some(after.clone());
3522        }
3523        self.state.operator_fixes[request_index].fix = Some(fix);
3524        // Saved again now that the fixer's own outcome is final, on top of
3525        // the save right after the request was first pushed above.
3526        self.state.save()?;
3527
3528        if committed {
3529            self.state.event(
3530                "fix",
3531                format!(
3532                    "operator fix committed {}; opening a follow-up review-only run",
3533                    short(&after)
3534                ),
3535            );
3536            match Self::review(&self.state.repo, &winner.branch, self.state.config.clone()).await {
3537                Ok(mut follow_up) => {
3538                    follow_up.state.event(
3539                        "start",
3540                        format!(
3541                            "requested by an operator fix on run {} for finding(s) {}",
3542                            self.state.id,
3543                            self.state.operator_fixes[request_index]
3544                                .findings
3545                                .iter()
3546                                .map(|f| f.id.as_str())
3547                                .collect::<Vec<_>>()
3548                                .join(", "),
3549                        ),
3550                    );
3551                    follow_up.state.save()?;
3552                    let follow_up_id = follow_up.state.id.clone();
3553                    if let Err(e) = follow_up.execute().await {
3554                        self.state.event(
3555                            "fix",
3556                            format!(
3557                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
3558                            ),
3559                        );
3560                    }
3561                    self.state.operator_fixes[request_index].follow_up_review_run =
3562                        Some(follow_up_id);
3563                }
3564                Err(e) => {
3565                    self.state.event(
3566                        "fix",
3567                        format!("committed the fix but could not open a follow-up review: {e:#}"),
3568                    );
3569                }
3570            }
3571            self.state.save()?;
3572        }
3573
3574        Ok(())
3575    }
3576
3577    // --------------------------------------------------------------- review
3578
3579    async fn review_loop(&mut self) -> Result<()> {
3580        // A base that would not rebase is a person's decision, not a review
3581        // round: nothing here would change the answer, and reviewers and a
3582        // fixer would be spending real budget on a tree that cannot land
3583        // regardless of what they find.
3584        if self
3585            .state
3586            .base_sync
3587            .as_ref()
3588            .is_some_and(|s| s.conflict.is_some())
3589        {
3590            return Ok(());
3591        }
3592        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3593        // agent files with `magi task add` name the run that paid for it. The
3594        // prompt overlay is cloned alongside it because the waves borrow it
3595        // while `self` is mutably borrowed by the node's own bookkeeping.
3596        let run_id = self.state.id.clone();
3597        let prompts = self.state.config.prompts.clone();
3598        let Some(winner) = self.state.winner().cloned() else {
3599            return Ok(());
3600        };
3601        let max_rounds = self.state.config.graph.review_rounds;
3602        // A clean round, an exhausted round budget, or a stalled tree (see
3603        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
3604        // they are recorded — recomputed here, not read off `status`, so a
3605        // reentry into a run that already stopped restates the identical
3606        // verdict instead of silently handing back whatever an earlier node
3607        // in this same walk clobbered `status` to (a solo-candidate
3608        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
3609        // below runs an empty range once the budget is spent, and would
3610        // otherwise fall through without touching `status` at all.
3611        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
3612            self.state.status = status;
3613            self.state.save()?;
3614            return Ok(());
3615        }
3616        self.state.status = RunStatus::Reviewing;
3617        // A last recorded round whose own verification never resolved
3618        // (`ResourceBlocked` — the shared build cache, not the patch) is
3619        // never a concluded round, whatever the round budget says: starting
3620        // a fresh round on top of it would spend a whole new reviewer wave
3621        // re-reading an unchanged patch instead of just retrying the one
3622        // check that actually needs it, and once the budget is spent the
3623        // loop below has nothing left to do at all (its range is empty).
3624        // Retry that check directly instead, exactly the same retry
3625        // `stop_reviewing` already does for its own catch-up case.
3626        if self
3627            .state
3628            .reviews
3629            .last()
3630            .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
3631        {
3632            let shell = self.state.config.shell();
3633            return self
3634                .stop_reviewing(
3635                    "the last round's own verification never resolved",
3636                    &shell,
3637                    &winner.worktree,
3638                )
3639                .await;
3640        }
3641
3642        let repo = self.state.repo.clone();
3643        let root = self.state.worktree_root();
3644        let language = self.state.config.graph.language.clone();
3645        let sessions = self.state.config.graph.sessions;
3646        let artifacts = agent::artifacts_dir(&self.state.dir());
3647        let base = self.landing_base();
3648        let base_short = short(&base);
3649        let reviewers = self.roles.reviewers.clone();
3650        let shell = self.state.config.shell();
3651
3652        for round in (self.state.reviews.len() + 1)..=max_rounds {
3653            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3654            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
3655            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
3656            // The prior round's own record, already persisted — never a
3657            // hand-carried variable of just its failing output: that is
3658            // exactly what let a round's e2e result drift out of sync with
3659            // which commit it was actually about (see `SCHEMA`'s doc for
3660            // schema 8). Judged against `head`, the commit reviewers are
3661            // about to look at now, so the summary always reads as "an
3662            // earlier head" here — this round's own patch has not been
3663            // checked yet.
3664            let prev_verification = self
3665                .state
3666                .reviews
3667                .last()
3668                .and_then(|r| r.verification_summary(&head));
3669
3670            // Each reviewer gets its own detached checkout of exactly this
3671            // commit: nobody can perturb the winner's tree, and the fixer can
3672            // keep working without racing a reviewer.
3673            let mut jobs = Vec::new();
3674            for (r, spec) in reviewers.iter().cloned().enumerate() {
3675                let wt = root.join(format!("review-{}", r + 1));
3676                if wt.exists() {
3677                    git::reset_detached(&wt, &head).await?;
3678                } else {
3679                    git::worktree_add_detached(&repo, &wt, &head).await?;
3680                }
3681                let seat_key = format!("review-{}", r + 1);
3682                let seat = self.seat(&seat_key, &spec.id);
3683                jobs.push(SeatJob {
3684                    prompt: prompt::review(&prompt::ReviewCtx {
3685                        instruction: &self.state.instruction,
3686                        branch: &winner.branch,
3687                        base_short: &base_short,
3688                        stat: &stat,
3689                        patch: &patch,
3690                        verification: prev_verification.as_ref(),
3691                        reviewers: reviewers.len(),
3692                        round,
3693                        rounds: max_rounds,
3694                        // A review-only run has no rankings, so nothing
3695                        // competed for this patch and the reviewer is told so.
3696                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
3697                        lens: Lens::for_seat(r),
3698                        language: &language,
3699                    }),
3700                    spec,
3701                    seat,
3702                    cwd: wt,
3703                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
3704                    allow_write: false,
3705                    sessions,
3706                    artifacts: artifacts.clone(),
3707                    stem: format!("review-{round}-{}", r + 1),
3708                });
3709            }
3710
3711            self.state.event(
3712                "review",
3713                format!(
3714                    "round {round}: {} reviewers on {}",
3715                    jobs.len(),
3716                    short(&head)
3717                ),
3718            );
3719            let mut quota_losses = Vec::new();
3720            let review_retries = self.state.config.graph.retries;
3721            let review_cache = self.state.config.cache_dir();
3722            let ctx = WaveCtx {
3723                run: &run_id,
3724                node: "review",
3725                prompts: &prompts,
3726                cache: review_cache.as_deref(),
3727                round: Some(round),
3728            };
3729            let results = ask_json_wave::<Review>(
3730                jobs,
3731                Arc::clone(&self.sem),
3732                review_retries,
3733                &ctx,
3734                &mut quota_losses,
3735                &mut self.state,
3736                &|_: &Review| Ok(()),
3737            )
3738            .await;
3739            // Counted before the move below: how many of *this* round's
3740            // reviewer seats were lost to their own rate limit, as opposed to
3741            // a crash, a timeout, or unparsable output — see `round_is_clean`.
3742            let round_quota_missing = quota_losses.len();
3743            self.state.quota.extend(quota_losses);
3744
3745            let mut records = Vec::new();
3746            let mut all_findings = Vec::new();
3747            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
3748                let agent_id = seat.agent.clone();
3749                self.state.seats.insert(seat.key.clone(), seat);
3750                let mut record = ReviewRecord {
3751                    reviewer: r + 1,
3752                    agent: agent_id,
3753                    summary: String::new(),
3754                    findings: Vec::new(),
3755                    vote: None,
3756                    failed: None,
3757                    duration_ms: 0,
3758                    // Set for both outcomes: `failed: Some(_)` with
3759                    // `attempts > 0` is a seat every retry still lost, not a
3760                    // recovered one — only `failed: None` with `attempts > 0`
3761                    // reads as "answered after a nudge" (see this field's own
3762                    // doc).
3763                    attempts,
3764                };
3765                match res {
3766                    Ok((review, out)) => {
3767                        // Sanitized here, at the point every other piece of
3768                        // agent prose in this file is (candidate summaries,
3769                        // deliberation turns, vote reasons): a reviewer's own
3770                        // words are the one thing about it that could name
3771                        // it, and reconsideration below broadcasts this same
3772                        // summary and these same findings to every other
3773                        // seat on the panel.
3774                        record.summary =
3775                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
3776                        record.vote = Some(review.vote);
3777                        record.duration_ms = out.duration_ms;
3778                        for (n, mut f) in review.findings.into_iter().enumerate() {
3779                            // ids are magi's, never the agent's: the fixer's
3780                            // adoption report is keyed by them.
3781                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
3782                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
3783                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
3784                            // `file` is agent-supplied prose too, never
3785                            // checked against the real tree — the same
3786                            // exposure `title`/`detail` above have, just in
3787                            // a field easy to forget because it looks like a
3788                            // path rather than free text.
3789                            f.file = f
3790                                .file
3791                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
3792                            all_findings.push(f.clone());
3793                            record.findings.push(f);
3794                        }
3795                        self.state.event(
3796                            "review",
3797                            format!(
3798                                "round {round}: reviewer {} voted {} with {} finding(s)",
3799                                r + 1,
3800                                review.vote.label(),
3801                                record.findings.len()
3802                            ),
3803                        );
3804                    }
3805                    Err(e) => {
3806                        record.failed = Some(e.to_string());
3807                        self.state.event(
3808                            "review",
3809                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
3810                        );
3811                    }
3812                }
3813                records.push(record);
3814            }
3815
3816            // Tally the round's votes and, if they split, spend the one
3817            // round of reconsideration the split -> deliberate -> revote
3818            // shape `judge`/`vote` use for the panel, sized down to what a
3819            // read-only review round can afford: one round, and a revote
3820            // rather than an argument, because the panel already wrote its
3821            // reasoning down as findings the first time around.
3822            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
3823            let vote_split =
3824                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
3825            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
3826            if vote_split {
3827                self.state.event(
3828                    "review",
3829                    format!(
3830                        "round {round}: votes split ({}) — one round of reconsideration",
3831                        initial_votes
3832                            .iter()
3833                            .map(|v| v.label())
3834                            .collect::<Vec<_>>()
3835                            .join(", ")
3836                    ),
3837                );
3838                // Seats read every seat's findings and votes, still numbered
3839                // and never named — the same anonymity `review` itself keeps.
3840                let panel: Vec<ReviewSeatReport<'_>> = records
3841                    .iter()
3842                    .filter_map(|r| {
3843                        r.vote.map(|vote| ReviewSeatReport {
3844                            reviewer: r.reviewer,
3845                            vote,
3846                            summary: &r.summary,
3847                            findings: &r.findings,
3848                        })
3849                    })
3850                    .collect();
3851
3852                let mut jobs = Vec::new();
3853                let mut seats_at = Vec::new();
3854                for (r, spec) in reviewers.iter().cloned().enumerate() {
3855                    // A seat with no initial vote has nothing to reconsider
3856                    // from and stays absent, the same as it stayed absent
3857                    // from `panel` above.
3858                    if records[r].vote.is_none() {
3859                        continue;
3860                    }
3861                    let wt = root.join(format!("review-{}", r + 1));
3862                    let seat_key = format!("review-{}", r + 1);
3863                    let seat = self.seat(&seat_key, &spec.id);
3864                    // A seat with no live session has already forgotten the
3865                    // initial review's prompt — restate the patch it is
3866                    // voting on, the same as `deliberate`/`vote` do for a
3867                    // judge in the same position.
3868                    let patch_ctx = if has_context(&spec, &seat, sessions) {
3869                        None
3870                    } else {
3871                        Some(ReviewPatch {
3872                            branch: &winner.branch,
3873                            base_short: &base_short,
3874                            stat: &stat,
3875                            patch: &patch,
3876                        })
3877                    };
3878                    let prompt = prompt::review_reconsider(&ReviewReconsiderCtx {
3879                        instruction: &self.state.instruction,
3880                        reviewer: r + 1,
3881                        lens: Lens::for_seat(r),
3882                        panel: &panel,
3883                        patch: patch_ctx,
3884                        round,
3885                        rounds: max_rounds,
3886                        language: &language,
3887                    });
3888                    jobs.push(SeatJob {
3889                        prompt,
3890                        spec,
3891                        seat,
3892                        cwd: wt,
3893                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
3894                        allow_write: false,
3895                        sessions,
3896                        artifacts: artifacts.clone(),
3897                        stem: format!("review-{round}-reconsider-{}", r + 1),
3898                    });
3899                    seats_at.push(r);
3900                }
3901
3902                let mut recon_quota_losses = Vec::new();
3903                let recon_cache = self.state.config.cache_dir();
3904                let recon_ctx = WaveCtx {
3905                    run: &run_id,
3906                    node: "review",
3907                    prompts: &prompts,
3908                    cache: recon_cache.as_deref(),
3909                    round: Some(round),
3910                };
3911                let recon_results = ask_json_wave::<ReviewRevote>(
3912                    jobs,
3913                    Arc::clone(&self.sem),
3914                    review_retries,
3915                    &recon_ctx,
3916                    &mut recon_quota_losses,
3917                    &mut self.state,
3918                    &|_: &ReviewRevote| Ok(()),
3919                )
3920                .await;
3921                self.state.quota.extend(recon_quota_losses);
3922
3923                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
3924                    let agent_id = seat.agent.clone();
3925                    self.state.seats.insert(seat.key.clone(), seat);
3926                    let mut rec = ReviewRevoteRecord {
3927                        reviewer: r + 1,
3928                        agent: agent_id,
3929                        vote: None,
3930                        reason: String::new(),
3931                        failed: None,
3932                    };
3933                    match res {
3934                        Ok((rv, _)) => {
3935                            rec.vote = Some(rv.vote);
3936                            rec.reason =
3937                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
3938                            self.state.event(
3939                                "review",
3940                                format!(
3941                                    "round {round}: reviewer {} revoted {}",
3942                                    r + 1,
3943                                    rv.vote.label()
3944                                ),
3945                            );
3946                        }
3947                        Err(e) => {
3948                            rec.failed = Some(e.to_string());
3949                            self.state.event(
3950                                "review",
3951                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
3952                            );
3953                        }
3954                    }
3955                    reconsideration.push(rec);
3956                }
3957            } else if initial_votes.len() > 1 {
3958                self.state.event(
3959                    "review",
3960                    format!(
3961                        "round {round}: votes agreed ({}) — no reconsideration",
3962                        initial_votes[0].label()
3963                    ),
3964                );
3965            }
3966
3967            // The final vote per seat is its revote where reconsideration
3968            // ran and answered, its initial vote otherwise — the same
3969            // fallback `tally` uses for a judge whose private vote failed.
3970            let final_votes: Vec<ReviewVote> = records
3971                .iter()
3972                .filter_map(|r| {
3973                    reconsideration
3974                        .iter()
3975                        .find(|rv| rv.reviewer == r.reviewer)
3976                        .and_then(|rv| rv.vote)
3977                        .or(r.vote)
3978                })
3979                .collect();
3980            let round_verdict = ReviewVote::worst(final_votes);
3981
3982            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
3983            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
3984            // A round that already has a blocking finding and a round left to
3985            // try is going back to the fixer no matter what `verify.e2e`
3986            // says, so running it first only spends the loop's slowest step
3987            // (minutes, for a Rust repo's full test suite) on a head about
3988            // to be rewritten. Deferred, never skipped: `verify.e2e` still
3989            // runs once a round has no blocking findings left (see
3990            // `round_is_clean`, which a deferred — empty — `e2e` can never
3991            // satisfy since `blocking` is nonzero whenever this branch is
3992            // taken), and `stop_reviewing` forces a real run before it will
3993            // ever read a deferred round as green.
3994            let defer_e2e =
3995                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
3996            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
3997                let reason =
3998                    format!("{blocking} blocking finding(s) already required a fix this round");
3999                self.state.event(
4000                    "verify",
4001                    format!(
4002                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
4003                         {}); it will run once a round has none left",
4004                        short(&head)
4005                    ),
4006                );
4007                (Vec::new(), false, true, Some(reason))
4008            } else {
4009                let e2e_commands = self.state.config.verify.e2e.clone();
4010                let cache_dir = self.state.config.cache_dir();
4011                let context = format!("round {round}");
4012                let (e2e, verify_retried) = with_cache_lease(
4013                    &mut self.state,
4014                    cache_dir.as_deref(),
4015                    "e2e",
4016                    "e2e",
4017                    &winner.worktree,
4018                    &head,
4019                    verify_timeout,
4020                    &context,
4021                    |state, budget| {
4022                        let shell = shell.clone();
4023                        let e2e_commands = e2e_commands.clone();
4024                        let worktree = winner.worktree.clone();
4025                        let context = context.clone();
4026                        async move {
4027                            run_e2e_with_retry(
4028                                state,
4029                                &shell,
4030                                &e2e_commands,
4031                                &worktree,
4032                                budget,
4033                                &context,
4034                            )
4035                            .await
4036                        }
4037                    },
4038                )
4039                .await;
4040                (e2e, verify_retried, false, None)
4041            };
4042
4043            let expected = records.len();
4044            let answered = records.iter().filter(|r| r.failed.is_none()).count();
4045            let incomplete = answered < expected;
4046            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
4047            let policy = self.state.config.graph.incomplete_review;
4048            let clean = round_is_clean(
4049                blocking,
4050                e2e_ok,
4051                answered,
4052                expected,
4053                round_quota_missing,
4054                policy,
4055            );
4056
4057            let mut round_record = ReviewRound {
4058                round,
4059                head: head.clone(),
4060                verified_head: None,
4061                verified_at: None,
4062                reviews: records,
4063                e2e,
4064                verify_retried,
4065                e2e_deferred,
4066                e2e_defer_reason,
4067                fix: None,
4068                blocking,
4069                answered,
4070                expected,
4071                clean,
4072                progressed: false,
4073                vote_split,
4074                reconsideration,
4075                verdict: round_verdict,
4076            };
4077            // Which commit and when magi actually attempted to check —
4078            // known the moment a command was dispatched against `head`,
4079            // whether or not it finished: a resource-blocked attempt still
4080            // targeted a specific commit at a specific time, and leaving
4081            // that unrecorded is exactly what made `verification_summary`
4082            // report a fresh attempt as "commit unknown ... recorded before
4083            // this was tracked", indistinguishable from a genuinely old,
4084            // untracked record. Only a deferred or unconfigured round never
4085            // ran at all and has nothing to record — see
4086            // `ReviewRound::verified_head`'s own doc.
4087            if !matches!(
4088                round_record.e2e_status(),
4089                E2eStatus::Deferred | E2eStatus::NotConfigured
4090            ) {
4091                round_record.verified_head = Some(head.clone());
4092                round_record.verified_at = Some(Timestamp::now());
4093            }
4094            let this_round_verification = round_record.verification_summary(&head);
4095
4096            if incomplete {
4097                let missing: Vec<String> = round_record
4098                    .reviews
4099                    .iter()
4100                    .filter(|r| r.failed.is_some())
4101                    .map(|r| format!("review-{}", r.reviewer))
4102                    .collect();
4103                self.state.event(
4104                    "review",
4105                    format!(
4106                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
4107                        missing.join(", ")
4108                    ),
4109                );
4110            }
4111
4112            if clean {
4113                self.state.event(
4114                    "review",
4115                    if incomplete && policy == IncompleteReviewPolicy::Warn {
4116                        format!(
4117                            "round {round}: clean (warn policy, incomplete panel) — no \
4118                             blocking findings from the seats that answered, verification green"
4119                        )
4120                    } else if incomplete {
4121                        format!(
4122                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
4123                             quorum) — no blocking findings from the seats that answered, \
4124                             verification green",
4125                            expected - answered
4126                        )
4127                    } else {
4128                        format!("round {round}: clean — no blocking findings, verification green")
4129                    },
4130                );
4131                self.state.reviews.push(round_record);
4132                self.state.status = RunStatus::Gating;
4133                self.state.save()?;
4134                return Ok(());
4135            }
4136
4137            // Nothing was raised and verification passed, but not every seat
4138            // answered and `round_is_clean` still refused to call it clean —
4139            // either a seat is missing for a reason other than its own quota
4140            // (a crash, a timeout, unparsable output — worth another try), or
4141            // every seat that could have answered lost its quota and nobody
4142            // is left to decide on: re-review rather than send the fixer
4143            // after a round with nothing to fix.
4144            if incomplete && blocking == 0 && e2e_ok {
4145                self.state.reviews.push(round_record);
4146                self.state.save()?;
4147                if round == max_rounds {
4148                    self.state.status = RunStatus::Blocked;
4149                    self.state.event(
4150                        "review",
4151                        format!(
4152                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
4153                             refusing to call it clean",
4154                            expected - answered
4155                        ),
4156                    );
4157                    return Ok(());
4158                }
4159                continue;
4160            }
4161
4162            // Nothing for the fixer to act on (`blocking == 0`) and the only
4163            // reason this round is not clean is that magi itself never got
4164            // a command to run — the shared build cache, not the patch (see
4165            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
4166            // the fixer would invite a change to appease contention that has
4167            // nothing to do with the diff, and would leave this attempt
4168            // sitting in the next round's prompt as if it were about an
4169            // earlier, superseded commit rather than what it actually is:
4170            // the same head, still waiting to be checked. Wait for it the
4171            // same way the final round's own contention is already handled,
4172            // whatever round this happens to be.
4173            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
4174                self.state.reviews.push(round_record);
4175                return self
4176                    .stop_reviewing(
4177                        "the round's own verification could not run",
4178                        &shell,
4179                        &winner.worktree,
4180                    )
4181                    .await;
4182            }
4183
4184            if round == max_rounds {
4185                self.state.reviews.push(round_record);
4186                return self
4187                    .stop_reviewing(
4188                        &format!(
4189                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
4190                        ),
4191                        &shell,
4192                        &winner.worktree,
4193                    )
4194                    .await;
4195            }
4196
4197            // Fix. The winner's own implementer seat continues its conversation:
4198            // the competition is over, so context is pure benefit now.
4199            let (fix_spec, fix_seat_key) = match &self.roles.fixer {
4200                Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
4201                _ => (
4202                    self.state
4203                        .config
4204                        .agent(&winner.agent)
4205                        .cloned()
4206                        .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
4207                    format!("impl-{}", winner.label),
4208                ),
4209            };
4210            let seat = self.seat(&fix_seat_key, &fix_spec.id);
4211            let blocking_findings: Vec<_> = all_findings
4212                .iter()
4213                .filter(|f| f.severity.blocks())
4214                .cloned()
4215                .collect();
4216            let job = SeatJob {
4217                prompt: prompt::fix(
4218                    &self.state.instruction,
4219                    &blocking_findings,
4220                    this_round_verification.as_ref(),
4221                    round,
4222                    max_rounds,
4223                    &language,
4224                ),
4225                spec: fix_spec.clone(),
4226                seat,
4227                cwd: winner.worktree.clone(),
4228                timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
4229                allow_write: true,
4230                sessions,
4231                artifacts: artifacts.clone(),
4232                stem: format!("fix-{round}"),
4233            };
4234            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
4235            let cache = self.state.config.cache_dir();
4236            let ctx = WaveCtx {
4237                run: &run_id,
4238                node: "fix",
4239                prompts: &prompts,
4240                cache: cache.as_deref(),
4241                round: Some(round),
4242            };
4243            let (seat, out) =
4244                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4245            let agent_id = seat.agent.clone();
4246
4247            let mut fix = FixRecord {
4248                agent: agent_id,
4249                addressed: Vec::new(),
4250                rejected: Vec::new(),
4251                notes: String::new(),
4252                committed: false,
4253                failed: None,
4254                duration_ms: 0,
4255                continuation: None,
4256            };
4257            let mut continuation = ContinuationRecord::not_needed();
4258            let mut final_seat = seat.clone();
4259            match out {
4260                AgentOutcome::Ok(o) => {
4261                    fix.duration_ms = o.duration_ms;
4262                    let parsed = verdict::extract_json::<FixReport>(&o.text);
4263                    // A parsed report standing next to a command this same
4264                    // reply's own CLI never confirmed the exit status of is
4265                    // not a resolved answer — the identical `CommandEvidence`
4266                    // `state.jobs` renders, read here instead of only on
4267                    // display, per the completion judgment and the shown
4268                    // record needing to agree.
4269                    let incomplete_reason = match &parsed {
4270                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4271                            "the reply parsed, but it reported a command whose own CLI never \
4272                             confirmed an exit status"
4273                                .to_owned(),
4274                        ),
4275                        Ok(_) => None,
4276                        Err(e) => Some(e.to_string()),
4277                    };
4278                    match incomplete_reason {
4279                        None => {
4280                            let report = parsed.expect("checked Ok above");
4281                            fix.addressed = report.addressed;
4282                            fix.rejected = report.rejected;
4283                            fix.notes =
4284                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
4285                        }
4286                        Some(reason) => {
4287                            let (resumed_seat, resolved, failure, cont) = self
4288                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
4289                                .await;
4290                            fix.duration_ms += cont.cumulative_wait_ms;
4291                            continuation = cont;
4292                            final_seat = resumed_seat;
4293                            match resolved {
4294                                Some(report) => {
4295                                    fix.addressed = report.addressed;
4296                                    fix.rejected = report.rejected;
4297                                    fix.notes = blind::sanitize_prose(
4298                                        &report.notes,
4299                                        &self.state.config.blind,
4300                                    );
4301                                }
4302                                None => fix.failed = failure,
4303                            }
4304                        }
4305                    }
4306                }
4307                // The CLI's raw error JSON is not a fix report to parse.
4308                AgentOutcome::Dropped(o) => {
4309                    fix.duration_ms = o.duration_ms;
4310                    let why = o
4311                        .dropped
4312                        .as_ref()
4313                        .map(|d| d.why.as_str())
4314                        .unwrap_or("the CLI ended the stream without delivering its answer");
4315                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4316                }
4317                AgentOutcome::Quota(o) => {
4318                    self.state.quota.push(QuotaLoss {
4319                        seat: final_seat.key.clone(),
4320                        node: "fix".to_owned(),
4321                        at: Timestamp::now(),
4322                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4323                    });
4324                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4325                }
4326                AgentOutcome::Failed(e) => fix.failed = Some(e),
4327            }
4328            fix.continuation = Some(continuation);
4329            self.state.seats.insert(final_seat.key.clone(), final_seat);
4330            git::commit_all(
4331                &winner.worktree,
4332                &format!("magi: review round {round} fixes (uncommitted work)"),
4333            )
4334            .await
4335            .ok();
4336            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
4337            fix.committed = after != before;
4338            // Judged by what `git` says moved against base, never by the
4339            // fixer's own `addressed`/`rejected` count — see
4340            // `ReviewRound::progressed`. Propagated with `?`, the same as the
4341            // `patch` snapshot above: swallowing this error would default
4342            // `diff_after` to empty, which almost always differs from a
4343            // non-empty `patch` and reads as "progressed" — exactly backwards
4344            // for a `git` failure the stagnation check cannot see through.
4345            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
4346            let progressed = diff_after != patch;
4347            let commit_note = if fix.committed {
4348                "committed"
4349            } else {
4350                "NO new commit"
4351            };
4352            let tree_note = if progressed {
4353                "changed vs base"
4354            } else {
4355                "unchanged vs base"
4356            };
4357            self.state.event(
4358                "fix",
4359                match &fix.failed {
4360                    // Distinct on purpose from "0 addressed, 0 rejected": the
4361                    // fixer's own diff still landed (blocking counts do keep
4362                    // falling round over round), only its adoption report did
4363                    // not come back, so this must never read like every
4364                    // finding was reviewed and declined.
4365                    Some(reason) => {
4366                        format!(
4367                            "round {round}: fixer's adoption report was lost ({reason}); \
4368                             {commit_note}, tree {tree_note}"
4369                        )
4370                    }
4371                    None => format!(
4372                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
4373                         {tree_note}{}",
4374                        fix.addressed.len(),
4375                        fix.rejected.len(),
4376                        if continuation.outcome == ContinuationOutcome::Resumed {
4377                            format!(
4378                                " (adoption report recovered after {} continuation(s))",
4379                                continuation.attempts
4380                            )
4381                        } else {
4382                            String::new()
4383                        },
4384                    ),
4385                },
4386            );
4387            round_record.fix = Some(fix);
4388            round_record.progressed = progressed;
4389            self.state.reviews.push(round_record);
4390            self.state.save()?;
4391
4392            // The fixer's own report never came back this round, even after
4393            // `continue_fix_report`'s own budget was spent on it — not an
4394            // ordinary "no report" (dropped stream, quota, plain failure),
4395            // which already reads that way and is left to the existing round
4396            // budget. Stopping here, rather than opening another round, is
4397            // what keeps a next reviewer/fixer wave from ever being
4398            // dispatched onto `winner.worktree` while whatever the seat's
4399            // last call may still have running there is unaccounted for: no
4400            // process liveness check exists (and none is being added — see
4401            // AGENTS.md/this task's own scope), so the only way to honour
4402            // "nothing starts before a valid report returns" is to not start
4403            // anything further on this worktree from this run at all.
4404            if matches!(
4405                continuation.outcome,
4406                ContinuationOutcome::Exhausted
4407                    | ContinuationOutcome::QuotaLost
4408                    | ContinuationOutcome::NoSession
4409            ) {
4410                return self
4411                    .stop_reviewing(
4412                        "the fixer's adoption report never came back, even after resuming its \
4413                         own seat; refusing to start another round against the same worktree \
4414                         while that is unresolved",
4415                        &shell,
4416                        &winner.worktree,
4417                    )
4418                    .await;
4419            }
4420
4421            let streak = self
4422                .state
4423                .reviews
4424                .iter()
4425                .rev()
4426                .take_while(|r| !r.progressed)
4427                .count();
4428            if streak >= STAGNANT_LIMIT {
4429                return self
4430                    .stop_reviewing(
4431                        &format!(
4432                            "the tree has not moved against base for {streak} round(s) in a row"
4433                        ),
4434                        &shell,
4435                        &winner.worktree,
4436                    )
4437                    .await;
4438            }
4439        }
4440        Ok(())
4441    }
4442
4443    /// Decide, from the last recorded round's own verification, whether
4444    /// stopping the review loop is a hand-off or a genuine block.
4445    ///
4446    /// Called once the loop has given up trying — the round budget is spent,
4447    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
4448    /// findings still open, never while a round is still clean or the
4449    /// incomplete-panel case handled inline above. Gate and e2e are facts
4450    /// about the tree; a lingering review finding is an opinion, and this
4451    /// workload's own `magi stats` puts reviewer precision low enough
4452    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
4453    /// must not by itself stand between a green, verified change and the
4454    /// human who decides what to do with it. A red e2e is not an opinion, so
4455    /// that case still blocks, with the failing command and a tail of its
4456    /// output recorded here rather than left in `run.json` for someone to go
4457    /// find.
4458    ///
4459    /// A round that deferred its own e2e (see [`Config::graph`]'s
4460    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
4461    /// only because nothing ran, and treating an empty list as a passing one
4462    /// here is exactly the "deferred painted green" bug this function exists
4463    /// to not have. When the last round's own verification never resolved —
4464    /// deferred on purpose, or a real attempt the shared build cache blocked
4465    /// — this makes (or retries) the real run, on the actual worktree this
4466    /// loop is about to stop touching, before deciding anything. A
4467    /// resource-blocked attempt is likewise never read as either green or
4468    /// red: it is evidence about the machine, not the patch (see
4469    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
4470    /// blocked cache leaves this call without deciding rather than guessing
4471    /// — the caller retries on a later reentry.
4472    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
4473        let round_idx = self.state.reviews.len() - 1;
4474        // A deferred round and a resource-blocked one are the same shape
4475        // here: neither has a real result yet, and both get one more
4476        // attempt. Read off `e2e_status` — the single source for this —
4477        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
4478        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
4479        // still retries instead of being read as a settled result the
4480        // instant it stops being empty.
4481        let needs_catchup_run = matches!(
4482            self.state.reviews[round_idx].e2e_status(),
4483            E2eStatus::Deferred | E2eStatus::ResourceBlocked
4484        );
4485        if needs_catchup_run {
4486            let round = self.state.reviews[round_idx].round;
4487            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4488            let commands = self.state.config.verify.e2e.clone();
4489            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
4490            let cache_dir = self.state.config.cache_dir();
4491            let context = format!(
4492                "round {round}: verification unresolved, catching up before the final decision"
4493            );
4494            let (outcomes, verify_retried) = with_cache_lease(
4495                &mut self.state,
4496                cache_dir.as_deref(),
4497                "e2e",
4498                "e2e",
4499                worktree,
4500                &attempted_head,
4501                timeout,
4502                &context,
4503                |state, budget| {
4504                    let shell = shell.to_vec();
4505                    let commands = commands.clone();
4506                    let context = context.clone();
4507                    async move {
4508                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
4509                            .await
4510                    }
4511                },
4512            )
4513            .await;
4514            let last = &mut self.state.reviews[round_idx];
4515            last.e2e = outcomes;
4516            last.verify_retried = verify_retried;
4517            // Always the commit and time this attempt actually targeted,
4518            // whether or not it happens to equal the reviewed `head` and
4519            // whether or not a command finished — see
4520            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
4521            // attempt is recorded too, so a later reader sees "attempted
4522            // again at T2" rather than silence.
4523            last.verified_head = Some(attempted_head);
4524            last.verified_at = Some(Timestamp::now());
4525            if verify_inconclusive(&last.e2e) {
4526                // Still not a real result: `e2e_deferred` is left exactly
4527                // as it was, so `needs_catchup_run` above reads
4528                // `ResourceBlocked` (via `e2e_status`, which checks
4529                // `resource_blocked` before `e2e_deferred`) and retries
4530                // again on the next reentry, rather than recording
4531                // contention as a red e2e and blocking the run on it.
4532                self.state.save()?;
4533                return Ok(());
4534            }
4535            last.e2e_deferred = false;
4536        }
4537        let last = &self.state.reviews[round_idx];
4538        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
4539
4540        match last.e2e_status() {
4541            E2eStatus::Failed => {
4542                let red: Vec<String> = last
4543                    .e2e
4544                    .iter()
4545                    .filter(|o| !o.ok())
4546                    .map(|o| {
4547                        format!(
4548                            "`{}` -> {:?}\n{}",
4549                            o.command,
4550                            o.code,
4551                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
4552                        )
4553                    })
4554                    .collect();
4555                self.state
4556                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
4557                self.state.status = RunStatus::Blocked;
4558            }
4559            // `needs_catchup_run` above already retried once this call; if
4560            // it is still blocked, this is magi's own admission it could
4561            // not get a command to run, never a verdict on the patch — the
4562            // run is left exactly where a later reentry can retry again.
4563            E2eStatus::ResourceBlocked => {
4564                self.state.event(
4565                    "review",
4566                    format!(
4567                        "{why}; e2e could not run (shared build cache unavailable); not \
4568                         deciding yet"
4569                    ),
4570                );
4571            }
4572            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
4573                self.state.event(
4574                    "review",
4575                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
4576                );
4577                self.state.status = RunStatus::Gating;
4578            }
4579        }
4580        self.state.save()?;
4581        Ok(())
4582    }
4583
4584    // ----------------------------------------------------------------- gate
4585
4586    async fn gate(&mut self) -> Result<()> {
4587        // Judged by the review record itself, not by `status`: a solo
4588        // candidate's `judge`/`deliberate` skip rewrites `status` on every
4589        // reentry (see `judge`), and trusting it here is exactly how a run
4590        // that exhausted its review budget got gated and merged a second
4591        // time around. `review_conclusion` recomputes the review loop's own
4592        // verdict from the round records themselves — `Gating` for a clean
4593        // round or a hand-off (see `stop_reviewing`), anything else means the
4594        // loop is still going or genuinely blocked.
4595        // A base the winner could not be replayed onto is a decision, not a
4596        // round: there is no landing tree to gate. Read as its own record for
4597        // the same reason the review verdict is.
4598        if self.state.status == RunStatus::Failed
4599            || self
4600                .state
4601                .base_sync
4602                .as_ref()
4603                .is_some_and(|s| s.conflict.is_some())
4604            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
4605                != Some(RunStatus::Gating)
4606        {
4607            return Ok(());
4608        }
4609        if self.state.gate_ran {
4610            // `review_loop` derives its conclusion from the clean review
4611            // record on every reentry and therefore puts a completed run back
4612            // in `Gating`. A recorded gate is a stronger, terminal fact:
4613            // retain its original command output (or lack of any, for a repo
4614            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
4615            // doc) and restore `Blocked` on a real failure rather than
4616            // pretending the command is still running or running it a second
4617            // time. `gate_ran == false` remains the only shape — unattempted,
4618            // or a resource-blocked retry — that may still need to execute a
4619            // command.
4620            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
4621                self.state.status = RunStatus::Blocked;
4622                self.state.save()?;
4623            }
4624            return Ok(());
4625        }
4626        let Some(winner) = self.state.winner().cloned() else {
4627            return Ok(());
4628        };
4629        self.state.status = RunStatus::Gating;
4630        let shell = self.state.config.shell();
4631        let gate_commands = self.state.config.verify.gate.clone();
4632        // Zero commands has nothing to run and nothing that could touch the
4633        // shared build cache, so it never needs a lease: `Config::cache_dir`
4634        // is derived from `verify.e2e` too, so a repo with no `verify.gate`
4635        // commands but a `CARGO_TARGET_DIR`-using `verify.e2e` would
4636        // otherwise queue behind an unrelated run's lease and come back
4637        // resource-blocked - `gate_ran` would stay false on nothing but
4638        // cache contention, for a step that had nothing to check in the
4639        // first place.
4640        let outcomes = if gate_commands.is_empty() {
4641            Vec::new()
4642        } else {
4643            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4644            let cache_dir = self.state.config.cache_dir();
4645            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4646            let (outcomes, _) = with_cache_lease(
4647                &mut self.state,
4648                cache_dir.as_deref(),
4649                "gate",
4650                "gate",
4651                &winner.worktree,
4652                &head,
4653                timeout,
4654                "final gate",
4655                |state, budget| {
4656                    let shell = shell.clone();
4657                    let gate_commands = gate_commands.clone();
4658                    let worktree = winner.worktree.clone();
4659                    async move {
4660                        let (outcomes, timed_out_pids) = run_commands(
4661                            state,
4662                            "gate",
4663                            "gate",
4664                            0,
4665                            &shell,
4666                            &gate_commands,
4667                            &worktree,
4668                            budget,
4669                        )
4670                        .await;
4671                        (outcomes, false, timed_out_pids)
4672                    }
4673                },
4674            )
4675            .await;
4676            outcomes
4677        };
4678        if outcomes.is_empty() {
4679            // Nothing configured to check — distinct from every other
4680            // silence in this run's event log, since an empty `gate` alone
4681            // no longer says whether the gate ran at all (see
4682            // `RunState::gate_ran`'s own doc).
4683            self.state.event(
4684                "gate",
4685                "no gate commands configured; nothing to check, passing",
4686            );
4687        }
4688        for o in &outcomes {
4689            self.state.event(
4690                "gate",
4691                format!(
4692                    "`{}` -> {}",
4693                    o.command,
4694                    if o.ok() {
4695                        "pass".to_owned()
4696                    } else {
4697                        format!(
4698                            "FAIL ({:?})\n{}",
4699                            o.code,
4700                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
4701                        )
4702                    }
4703                ),
4704            );
4705        }
4706        // A resource-blocked outcome means the gate command never actually
4707        // ran - the shared build cache could not be acquired or confirmed
4708        // fresh in time - which is evidence about the machine, not about the
4709        // tree (see `CommandOutcome::resource_blocked`'s own doc). Recording
4710        // it as a red gate would mark a run `Blocked` on nothing but
4711        // contention magi has already logged above; leaving `self.state.gate`
4712        // empty and `self.state.gate_ran` false instead keeps the shape this
4713        // function already treats as "still needs to run" (see the
4714        // early-return above), so the next call retries the command rather
4715        // than concluding anything.
4716        if verify_inconclusive(&outcomes) {
4717            self.state.save()?;
4718            return Ok(());
4719        }
4720        let passed = outcomes.iter().all(CommandOutcome::ok);
4721        self.state.gate = outcomes;
4722        self.state.gate_ran = true;
4723        if !passed {
4724            self.state.status = RunStatus::Blocked;
4725            self.state.event("gate", "gate failed; not merging");
4726        }
4727        self.state.save()?;
4728        Ok(())
4729    }
4730
4731    // ---------------------------------------------------------------- merge
4732
4733    async fn merge(&mut self) -> Result<()> {
4734        // Same reasoning as `gate`: ask the review and gate records directly
4735        // rather than `status`, which a solo-candidate `judge`/`deliberate`
4736        // skip can rewrite on reentry to something that no longer says
4737        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
4738        // so a hand-off (open findings, green verification) reaches merge
4739        // exactly like a genuinely clean round does.
4740        //
4741        // A run resumed mid-`land` never reaches here at all: `execute`
4742        // recognises `RunStatus::Landing` before it even calls `prep`, and
4743        // routes straight to `run_land` instead. That has to happen a level
4744        // up from this function, not with a check in here, because
4745        // `review_loop`'s own status recomputation (see its doc) runs
4746        // *before* `merge` on every reentry and would otherwise overwrite
4747        // the `Landing` marker with `Gating` before this node ever saw it.
4748        if self
4749            .state
4750            .base_sync
4751            .as_ref()
4752            .is_some_and(|s| s.conflict.is_some())
4753            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
4754                != Some(RunStatus::Gating)
4755            // `gate_ran == false` is not "passed" - `gate` leaves it false
4756            // both before it has ever run and when its last attempt was
4757            // resource-blocked (see `Runner::gate`'s own doc), and neither is
4758            // permission to merge on nothing but the review record. Only a
4759            // gate that actually ran - zero commands configured and
4760            // vacuously passed, or one or more that all exited 0 - may
4761            // proceed; `RunState::gate_status` is the single place that
4762            // reading is computed.
4763            || !self.state.gate_status().ok()
4764        {
4765            return Ok(());
4766        }
4767        // This node's own record, not `status`: `status == Ready` is not
4768        // unique to the harmless `MergeMode::None` path this line was
4769        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
4770        // run's PR was closed without merging — and on that run `mode` is
4771        // still `Pr`, so a reentry that fell through here would push and
4772        // open a second pull request. `self.state.merge` is set exactly once
4773        // this node (or `land`) has already produced a verdict, under every
4774        // mode, which is what "already done" actually means here.
4775        if self.state.merge.is_some() {
4776            return Ok(());
4777        }
4778        let Some(winner) = self.state.winner().cloned() else {
4779            return Ok(());
4780        };
4781        let repo = self.state.repo.clone();
4782        let base = self.state.base_branch.clone();
4783        let mode = self.state.config.merge.mode;
4784        let style = self.state.config.merge.style;
4785        let message = pr_body(&self.state, winner.label);
4786
4787        let outcome = match mode {
4788            MergeMode::None => MergeOutcome {
4789                mode,
4790                ok: true,
4791                detail: manual_merge_command(style, &repo, &winner.branch, &message),
4792            },
4793            MergeMode::Local => {
4794                let on = git::current_branch(&repo).await?;
4795                if on.as_deref() != Some(base.as_str()) {
4796                    MergeOutcome {
4797                        mode,
4798                        ok: false,
4799                        detail: format!(
4800                            "{} has {} checked out, not the base branch {base}",
4801                            repo.display(),
4802                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
4803                        ),
4804                    }
4805                } else if !git::is_clean(&repo).await? {
4806                    MergeOutcome {
4807                        mode,
4808                        ok: false,
4809                        detail: format!("{} is dirty; refusing to merge", repo.display()),
4810                    }
4811                } else {
4812                    let out = match style {
4813                        MergeStyle::Merge => {
4814                            git::merge_no_ff(&repo, &winner.branch, &message).await?
4815                        }
4816                        MergeStyle::Squash => {
4817                            git::merge_squash(&repo, &winner.branch, &message).await?
4818                        }
4819                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
4820                    };
4821                    MergeOutcome {
4822                        mode,
4823                        ok: out.ok(),
4824                        detail: if out.ok() { out.stdout } else { out.stderr },
4825                    }
4826                }
4827            }
4828            MergeMode::Pr => {
4829                let remote = self.state.config.merge.remote.clone();
4830                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
4831                if !pushed.ok() {
4832                    MergeOutcome {
4833                        mode,
4834                        ok: false,
4835                        detail: pushed.stderr,
4836                    }
4837                } else {
4838                    let out = gh_pr_create(&winner.worktree, &base, &winner.branch, &message).await;
4839                    match out {
4840                        Ok(url) => MergeOutcome {
4841                            mode,
4842                            ok: true,
4843                            detail: url,
4844                        },
4845                        Err(e) => MergeOutcome {
4846                            mode,
4847                            ok: false,
4848                            detail: e.to_string(),
4849                        },
4850                    }
4851                }
4852            }
4853        };
4854
4855        self.state.status = match (mode, outcome.ok) {
4856            (MergeMode::None, _) => RunStatus::Ready,
4857            (_, true) => RunStatus::Merged,
4858            (_, false) => RunStatus::Blocked,
4859        };
4860        self.state.event(
4861            "merge",
4862            format!(
4863                "{:?}: {}",
4864                mode,
4865                outcome.detail.lines().next().unwrap_or("")
4866            ),
4867        );
4868        self.state.merge = Some(outcome);
4869        self.state.save()?;
4870
4871        // The PR is open and the run would historically stop here, leaving the
4872        // operator to watch checks, feed review comments back to a fixer, and
4873        // merge. That was done by hand six times in one session before this
4874        // existed. Opt-in, because merging is the one irreversible thing magi
4875        // can do to a repository.
4876        if self.state.config.graph.land
4877            && mode == MergeMode::Pr
4878            && self.state.status == RunStatus::Merged
4879        {
4880            self.run_land().await?;
4881        }
4882        // `run_land` may have left `status` at `Landing` - still waiting on
4883        // CI or the owner's approval, not actually settled - so this has to
4884        // read whatever `status` ended up as here, not the `Merged` this
4885        // function set a few lines up.
4886        self.settle_questions();
4887        Ok(())
4888    }
4889
4890    /// Enter `land`.
4891    ///
4892    /// Shared between a fresh run's first pass through [`Runner::merge`] and
4893    /// a resumed run's re-entry. `land::land` itself is what serialises the
4894    /// two git-mutating moments inside the loop — the rebase push and
4895    /// `gh pr merge` — per repository (see its own doc); nothing here needs
4896    /// to hold a lock across the whole call, and doing so would serialise
4897    /// this run's CI wait against a *different* run's land-approval resume
4898    /// in the same repository, which is exactly the "must not wait on
4899    /// another task" property the daemon's slot-freeing exists to give.
4900    async fn run_land(&mut self) -> Result<()> {
4901        let url = self
4902            .state
4903            .merge
4904            .as_ref()
4905            .map(|m| m.detail.clone())
4906            .unwrap_or_default();
4907        let url = url.lines().next().unwrap_or("").trim().to_owned();
4908        if !url.starts_with("http") {
4909            return Ok(());
4910        }
4911        // A land failure is not a lost run: the work is on a branch and the
4912        // pull request is open, which is exactly where a human takes over.
4913        match land::land(&mut self.state, &url).await {
4914            Ok(pr) if self.state.parked => {
4915                // `land` already saved the parked marker; nothing here
4916                // overrides `status` back to a terminal value while an
4917                // approval is still outstanding.
4918                let _ = pr;
4919            }
4920            Ok(pr) => {
4921                self.state.status = match pr.state {
4922                    land::PrLifecycle::Merged => RunStatus::Merged,
4923                    _ => RunStatus::Blocked,
4924                };
4925                // Downstream of a confirmed merge only - see
4926                // `bump::should_release_bump`'s own doc for why this one
4927                // check covers all three of `land`'s success paths.
4928                // Best-effort: the run already landed, so a failure here
4929                // (the decision call, `gh`, `cargo`) is recorded and never
4930                // turns a landed run into a failed one.
4931                if bump::should_release_bump(self.state.status)
4932                    && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
4933                {
4934                    self.state
4935                        .event("bump", format!("release bump skipped: {e:#}"));
4936                }
4937                self.state.save()?;
4938            }
4939            Err(e) => {
4940                self.state.status = RunStatus::Blocked;
4941                self.state.event("land", format!("gave up: {e}"));
4942                self.state.save()?;
4943            }
4944        }
4945        Ok(())
4946    }
4947
4948    // -------------------------------------------------------------- helpers
4949
4950    /// Fetch or create a seat, keeping its conversation across nodes.
4951    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
4952        if let Some(existing) = self.state.seats.get(key)
4953            && existing.agent == agent
4954        {
4955            return existing.clone();
4956        }
4957        let fresh = SeatState::new(key, agent, self.state.seed);
4958        self.state.seats.insert(key.to_owned(), fresh.clone());
4959        fresh
4960    }
4961
4962    /// A candidate rendered for judging, with the leak policy applied.
4963    fn view(&self, c: &Candidate) -> CandidateView {
4964        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4965            .unwrap_or_default();
4966        let (patch, _) = blind::sanitize_patch(
4967            &format!("candidate {} patch", c.label),
4968            &raw,
4969            &self.state.config.blind,
4970        );
4971        CandidateView {
4972            label: c.label,
4973            branch: c.branch.clone(),
4974            summary: c.summary.clone(),
4975            stat: c.stat.clone(),
4976            patch,
4977        }
4978    }
4979
4980    /// The full candidate set as prompt text, for seats with no live session.
4981    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
4982        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
4983        prompt::judge(
4984            "(see above)",
4985            &views,
4986            self.roles.judges.len(),
4987            base_short,
4988            "en",
4989        )
4990    }
4991
4992    /// Anonymised transcript for judge `self_idx`.
4993    ///
4994    /// The initial rankings are always the opening statements. Seeding them
4995    /// only when no turn had been taken yet meant every judge after the first
4996    /// argued against a single voice instead of against the actual split — the
4997    /// disagreement is the information, so it is always on the table.
4998    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
4999        let mut turns = Vec::new();
5000        for j in &self.state.judgements {
5001            if j.ranking.is_empty() {
5002                continue;
5003            }
5004            let reasons = j
5005                .reasons
5006                .iter()
5007                .map(|(k, v)| format!("- {k}: {v}"))
5008                .collect::<Vec<_>>()
5009                .join("\n");
5010            turns.push(Turn {
5011                who: format!("Judge {} (opening ranking)", j.judge),
5012                is_self: j.judge == self_idx + 1,
5013                body: format!(
5014                    "Ranked {}{}{reasons}",
5015                    j.ranking.iter().collect::<String>(),
5016                    if reasons.is_empty() {
5017                        ""
5018                    } else {
5019                        ", because:\n"
5020                    }
5021                ),
5022            });
5023        }
5024        for t in self
5025            .state
5026            .deliberation
5027            .iter()
5028            .flat_map(|r| r.turns.iter())
5029            .chain(current)
5030        {
5031            turns.push(Turn {
5032                who: format!("Judge {}", t.judge),
5033                is_self: t.judge == self_idx + 1,
5034                body: t.body.clone(),
5035            });
5036        }
5037        turns
5038    }
5039}
5040
5041/// Does this seat still hold the context a follow-up prompt would rely on?
5042fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
5043    agent::has_session(spec.kind, seat, sessions)
5044}
5045
5046/// The next entry in `roster` after `start`, never wrapping back to the
5047/// front, whose id is not in `tried` yet.
5048///
5049/// Starts one past `start` rather than at the front of `roster`: `start` is
5050/// the seat's own original position, and a seat whose candidate slot already
5051/// sits on the roster's second entry must fall through to the third next, not
5052/// restart at the first — which is very likely a different candidate's own
5053/// agent already. Never wraps back past `start`, for the same reason: an
5054/// entry earlier in the roster than the seat's own position is almost
5055/// certainly some *other* candidate slot's own agent, and once the tail of
5056/// the roster is exhausted there are no more untried agents for *this* seat
5057/// to fall through to — the caller's fallback chain ends there, exactly as
5058/// "no further untried agents remain in the list for that seat" asks for.
5059///
5060/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
5061/// the same id twice (an operator's `roles.implementers` typo, or a
5062/// `[[agents]]` list reused across roles) must not let
5063/// [`Runner::resume_quota_losses`] retry that id forever — one forward pass
5064/// over `roster` either finds an untried id or runs out, so this always
5065/// terminates regardless of duplicates.
5066fn next_untried_implementer<'a>(
5067    roster: &'a [AgentSpec],
5068    start: usize,
5069    tried: &BTreeSet<String>,
5070) -> Option<&'a AgentSpec> {
5071    roster
5072        .get(start + 1..)?
5073        .iter()
5074        .find(|s| !tried.contains(&s.id))
5075}
5076
5077/// Did this reply report running a command whose own CLI never confirmed an
5078/// exit status?
5079///
5080/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
5081/// command *finished* (see that type's own doc), so this can only be `true`
5082/// for a command whose completion event carried no readable exit code — not
5083/// for one that simply is not mentioned at all. That is the one signal this
5084/// crate can read, from the same record `state.jobs` renders, about a reply
5085/// standing next to work its own CLI cannot vouch for finishing; it is
5086/// deliberately not a check on the exit code's *value* (a fixer legitimately
5087/// runs a command that fails mid-iteration before it succeeds) and not a
5088/// guess at a command still running in the background (which emits no event
5089/// at all, and so leaves no evidence here to find).
5090fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
5091    commands.iter().any(|c| c.exit_code.is_none())
5092}
5093
5094/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
5095/// trusted as a verified no-op — the adoption guard's own text-level half.
5096///
5097/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
5098/// timed out): a marker only earns the benefit of the doubt from a turn the
5099/// CLI itself vouches for finishing properly, the same house style
5100/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
5101/// *fix* report to for `commands`. A candidate that timed out, exited
5102/// non-zero, or left a command unconfirmed is read as the ordinary loss it
5103/// is, whatever prose it wrote — this returns `None` before it ever looks at
5104/// `text`. The remaining guards (the tree really is empty, the evidence is
5105/// non-empty) are the caller's: this only reads what the reply *claimed*.
5106fn verified_noop_claim(
5107    usable: bool,
5108    commands: &[agent::CommandEvidence],
5109    text: &str,
5110) -> Option<String> {
5111    (usable && !has_unconfirmed_command(commands))
5112        .then(|| verdict::verified_noop(text))
5113        .flatten()
5114}
5115
5116fn short(commit: &str) -> String {
5117    commit.chars().take(7).collect()
5118}
5119
5120fn make_executable(path: &Path) -> Result<()> {
5121    #[cfg(unix)]
5122    {
5123        use std::os::unix::fs::PermissionsExt as _;
5124        let mut perms = std::fs::metadata(path)?.permissions();
5125        perms.set_mode(0o755);
5126        std::fs::set_permissions(path, perms)?;
5127    }
5128    #[cfg(not(unix))]
5129    {
5130        let _ = path;
5131    }
5132    Ok(())
5133}
5134
5135/// What every seat in one batch shares: where the answers are attributed, the
5136/// prompt overlay they inherit, and the build cache they are told to use.
5137///
5138/// A struct rather than four more parameters: `wave` also needs the run's
5139/// state (to record who is answering right now) and the attempt number, and
5140/// eight positional arguments is both unreadable and a clippy error.
5141struct WaveCtx<'a> {
5142    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
5143    /// paid for it.
5144    run: &'a str,
5145    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
5146    node: &'a str,
5147    prompts: &'a Prompts,
5148    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
5149    cache: Option<&'a Path>,
5150    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
5151    /// `JobRecord::round`. `None` for every other node.
5152    round: Option<usize>,
5153}
5154
5155/// Run one job, honouring the parallelism budget.
5156async fn run_one(
5157    job: SeatJob,
5158    sem: Arc<Semaphore>,
5159    ctx: &WaveCtx<'_>,
5160    state: &mut RunState,
5161    attempt: usize,
5162) -> (SeatState, AgentOutcome) {
5163    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
5164        .await
5165        .pop()
5166        .expect("one job in, one result out");
5167    (seat, out)
5168}
5169
5170/// Run every job concurrently, capped by the semaphore, preserving order.
5171///
5172/// Every seat in the batch is recorded into [`RunState::active`] before the
5173/// wave starts and cleared as each answer lands, so the run's own record says
5174/// who is still being waited on rather than only who finished.
5175async fn wave(
5176    jobs: Vec<SeatJob>,
5177    sem: Arc<Semaphore>,
5178    ctx: &WaveCtx<'_>,
5179    state: &mut RunState,
5180    attempt: usize,
5181) -> Vec<(usize, SeatState, AgentOutcome)> {
5182    let WaveCtx {
5183        run,
5184        node,
5185        prompts,
5186        cache,
5187        round,
5188    } = *ctx;
5189    for job in &jobs {
5190        state.seat_started(node, &job.seat.key, job.timeout, attempt);
5191    }
5192    if let Err(e) = state.save() {
5193        // A failed persist of "who is answering right now" must not abort the
5194        // wave: the seats are already being asked, and the alternative is
5195        // losing the answers to save a status line nobody may even be
5196        // watching.
5197        tracing::warn!("could not persist in-progress seats: {e:#}");
5198    }
5199    // Hold the shared build cache's lease for the whole batch, not per job:
5200    // several candidates (an implement wave) or a fixer legitimately share
5201    // one cache concurrently within this run, and that stays untouched — a
5202    // single lease taken once for the whole wave and released once it is
5203    // done is what stops a *different* borrower (another run's own wave, its
5204    // e2e/gate, a human's `magi review`) from interleaving a build into the
5205    // same directory while this one is in flight. Best-effort, not
5206    // all-or-nothing: a wave that cannot get the lease within its own
5207    // longest job's budget still runs — an hour of paid implementer calls is
5208    // not thrown away over cache contention — but every write-allowed seat
5209    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
5210    // below), the same fallback a read-only seat always gets, rather than
5211    // building into a directory this run was never granted. The identity
5212    // record is still invalidated below either way, so the next tracked
5213    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
5214    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
5215    let wait_started = Instant::now();
5216    let cache_guard = if let Some(cache_dir) = cache {
5217        if jobs_had_a_writer {
5218            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
5219            let budget = jobs
5220                .iter()
5221                .map(|j| j.timeout)
5222                .max()
5223                .unwrap_or(Duration::from_secs(60));
5224            acquire_cache_lease(state, cache_dir, &owner, budget, node)
5225                .await
5226                .ok()
5227        } else {
5228            None
5229        }
5230    } else {
5231        None
5232    };
5233    // Carved out of each job's own budget, not added on top of it: a seat
5234    // that waited behind the lease must not also get its full timeout
5235    // afterward, or a run contended on the cache could double the time it
5236    // spends per wave. `saturating_sub` floors at zero rather than
5237    // wrapping - a job whose whole budget was spent waiting starts with
5238    // none left, which is the honest number, not a free minimum.
5239    let waited_for_lease = wait_started.elapsed();
5240    let mut set = tokio::task::JoinSet::new();
5241    let overlay = prompts.overlay(node);
5242    for (i, mut job) in jobs.into_iter().enumerate() {
5243        job.timeout = job.timeout.saturating_sub(waited_for_lease);
5244        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
5245        if cache.is_some() {
5246            job.prompt.push('\n');
5247            job.prompt
5248                .push_str(&prompt::build_cache_note(node, job.allow_write));
5249        }
5250        let sem = Arc::clone(&sem);
5251        let run = run.to_owned();
5252        let node = node.to_owned();
5253        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
5254        // `prompt::build_cache_note`'s doc for why setting it anyway is
5255        // exactly how a sandboxed reviewer's write refusal got reported as a
5256        // defect in the patch, not a property of its own seat. And a
5257        // write-allowed one is handed it only when the lease above was
5258        // actually acquired: a wave that could not get it (`cache_guard` is
5259        // `None`, see its own comment) must not send seats to build into a
5260        // directory this run does not hold - that is the exact concurrent,
5261        // unmanaged-write race this module exists to prevent, not something
5262        // "proceeding anyway" is allowed to reintroduce.
5263        let cache = cache
5264            .filter(|_| job.allow_write && cache_guard.is_some())
5265            .map(Path::to_path_buf);
5266        set.spawn(async move {
5267            let _permit = sem.acquire().await;
5268            let mut seat = job.seat;
5269            let out = agent::invoke(
5270                &job.spec,
5271                &mut seat,
5272                &Invocation {
5273                    cwd: &job.cwd,
5274                    prompt: &job.prompt,
5275                    timeout: job.timeout,
5276                    allow_write: job.allow_write,
5277                    sessions: job.sessions,
5278                    artifacts: &job.artifacts,
5279                    stem: &job.stem,
5280                    run: &run,
5281                    node: &node,
5282                    cache_dir: cache.as_deref(),
5283                    attachments: &[],
5284                },
5285            )
5286            .await;
5287            let out = match out {
5288                Ok(o) if o.usable() => AgentOutcome::Ok(o),
5289                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
5290                // Billed work the CLI failed to hand over is not an ordinary
5291                // failure, but its text is the CLI's raw error JSON, not an
5292                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
5293                // read it as one by forgetting to check. `usable()` is always
5294                // false here (dropped implies an empty response), so this has
5295                // to be checked before the catch-all `Failed` below or the
5296                // one shape this exists for is lost with the rest.
5297                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
5298                Ok(o) if o.timed_out => AgentOutcome::Failed("timed out".to_owned()),
5299                Ok(o) => AgentOutcome::Failed(format!(
5300                    "exited with {:?} and no usable output",
5301                    o.exit_code
5302                )),
5303                Err(e) => AgentOutcome::Failed(e.to_string()),
5304            };
5305            (i, seat, out)
5306        });
5307    }
5308    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
5309    while let Some(joined) = set.join_next().await {
5310        let (i, seat, out) = match joined {
5311            Ok(v) => v,
5312            // No seat to clear: a panicked task never reported which one it
5313            // was. The defensive sweep below this loop is what stops that
5314            // seat's `active` entry from surviving forever.
5315            Err(e) => {
5316                tracing::error!("agent task panicked: {e}");
5317                continue;
5318            }
5319        };
5320        state.seat_finished(&seat.key);
5321        record_jobs(state, node, round, &seat.key, &out);
5322        if let Err(e) = state.save() {
5323            tracing::warn!("could not persist a seat's completion: {e:#}");
5324        }
5325        if collected.len() <= i {
5326            collected.resize_with(i + 1, || None);
5327        }
5328        collected[i] = Some((i, seat, out));
5329    }
5330    // Belt-and-braces for the panic branch above: every seat this exact batch
5331    // started shares this `(node, attempt)` pair, and every seat that finished
5332    // normally already cleared itself, so anything left tagged with it here
5333    // can only be a panicked task's leftover. Cleared unconditionally rather
5334    // than left to read as still answering forever.
5335    if state
5336        .active
5337        .values()
5338        .any(|a| a.node == node && a.attempt == attempt)
5339    {
5340        state
5341            .active
5342            .retain(|_, a| !(a.node == node && a.attempt == attempt));
5343        if let Err(e) = state.save() {
5344            tracing::warn!("could not persist the end of a wave: {e:#}");
5345        }
5346    }
5347    // Whether or not the lease above was actually held, several worktrees
5348    // may just have built into the cache with nothing here able to name one
5349    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
5350    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
5351    // clean it might not have strictly needed; trusting a stale match would
5352    // cost it a wrong answer.
5353    if let Some(cache_dir) = cache
5354        && jobs_had_a_writer
5355    {
5356        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
5357    }
5358    if let Some(guard) = cache_guard {
5359        guard.release();
5360    }
5361    collected.into_iter().flatten().collect()
5362}
5363
5364/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
5365/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
5366/// this is data collection, not the fix-specific completion contract in
5367/// [`Runner::continue_fix_report`], and applies regardless of which node
5368/// asked.
5369///
5370/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
5371/// evidence from; `Failed` does not, and correctly contributes nothing — a
5372/// timeout or crash is not itself evidence about a command the seat may have
5373/// started.
5374fn record_jobs(
5375    state: &mut RunState,
5376    node: &str,
5377    round: Option<usize>,
5378    seat: &str,
5379    out: &AgentOutcome,
5380) {
5381    let commands: &[agent::CommandEvidence] = match out {
5382        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
5383        AgentOutcome::Failed(_) => &[],
5384    };
5385    let checked_at = Timestamp::now();
5386    for c in commands {
5387        state.jobs.push(JobRecord {
5388            node: node.to_owned(),
5389            round,
5390            seat: seat.to_owned(),
5391            id: c.id.clone(),
5392            description: c.description.clone(),
5393            checked_at,
5394            status: match c.exit_code {
5395                Some(0) => JobStatus::Completed,
5396                Some(_) => JobStatus::Failed,
5397                None => JobStatus::Unknown,
5398            },
5399            exit_code: c.exit_code,
5400            result_summary: c.result_summary.clone(),
5401            source: c.source.clone(),
5402        });
5403    }
5404}
5405
5406/// Is a review round clean, given how many reviewer seats answered against
5407/// how many the round expected?
5408///
5409/// A seat that never answered (timeout, crash, unparsable output) is not a
5410/// seat that read the patch and found nothing — treating it as such is
5411/// exactly the bug this function exists to close. Under the default `block`
5412/// policy a missing seat can never be clean; `warn` still requires the seats
5413/// that *did* answer to have found nothing blocking and verification to be
5414/// green.
5415///
5416/// `quota_missing` narrows that `block` default for exactly one cause of
5417/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
5418/// a session limit lifts by the very next round buys nothing — the seat is
5419/// asked again with the same quota — so once every missing seat is accounted
5420/// for by a quota loss (and at least one seat *did* answer, so a decision has
5421/// something to rest on) the round is decided on the panel that could answer,
5422/// same as `warn` would. A panel that lost every seat to quota is not
5423/// decided here: `answered == 0` falls through to the existing `block`
5424/// fallback so a fully collapsed panel still waits rather than landing on no
5425/// review at all.
5426fn round_is_clean(
5427    blocking: usize,
5428    e2e_ok: bool,
5429    answered: usize,
5430    expected: usize,
5431    quota_missing: usize,
5432    policy: IncompleteReviewPolicy,
5433) -> bool {
5434    if blocking != 0 || !e2e_ok {
5435        return false;
5436    }
5437    if answered == expected || policy == IncompleteReviewPolicy::Warn {
5438        return true;
5439    }
5440    answered > 0 && expected - answered <= quota_missing
5441}
5442
5443/// The review loop's own conclusion, derived entirely from its persisted
5444/// round records and the round budget that produced them — never from
5445/// `status`, so a reentry (or `gate`/`merge` reading it independently)
5446/// recomputes the identical answer regardless of what an earlier node in the
5447/// same walk, or a previous walk, did to `status`.
5448///
5449/// `None` while more rounds remain to try, including when review never ran
5450/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
5451/// gone clean, or the budget is spent, or the tree has stopped moving (see
5452/// [`STAGNANT_LIMIT`]), the answer is one of two things:
5453///
5454/// - An incomplete panel that raised nothing is missing input, not a
5455///   verified tree — never a hand-off candidate, whatever verification said
5456///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
5457/// - Otherwise, green e2e on the last round hands off (see
5458///   [`Runner::stop_reviewing`]); red e2e blocks.
5459///
5460/// A last round whose own verification is still `ResourceBlocked` — magi
5461/// itself never got a command to run, not evidence the patch is broken —
5462/// is neither: this returns `None` for it too, the same as "more rounds
5463/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
5464/// handling of that shape) instead of this cheap recomputation guessing a
5465/// verdict a real attempt never produced.
5466fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
5467    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
5468        return Some(RunStatus::Gating);
5469    }
5470    let last = reviews.last()?;
5471    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
5472    if reviews.len() < max_rounds && !stagnant {
5473        return None;
5474    }
5475    if last.incomplete() && last.blocking == 0 {
5476        return Some(RunStatus::Blocked);
5477    }
5478    if last.e2e_status() == E2eStatus::ResourceBlocked {
5479        return None;
5480    }
5481    Some(if last.e2e.iter().all(CommandOutcome::ok) {
5482        RunStatus::Gating
5483    } else {
5484        RunStatus::Blocked
5485    })
5486}
5487
5488/// How long a re-ask may take, given the budget the first attempt had.
5489///
5490/// A `nudged` retry is a request to restate an answer the seat has already
5491/// worked out: it carries no new work, so it does not deserve the original
5492/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
5493/// 133 seconds while a third sat for over ten minutes on a resumed session
5494/// holding 410 KB of prior output - and because the retry had inherited the
5495/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
5496/// judging round whose other seats were long finished.
5497///
5498/// A quarter of the budget, with a floor so that a deliberately short timeout
5499/// does not collapse to nothing. A retry that re-sends the whole prompt
5500/// (because the seat kept no context) is the original job again, and keeps the
5501/// original budget.
5502fn retry_budget(full: Duration, nudged: bool) -> Duration {
5503    if nudged {
5504        (full / 4).max(Duration::from_secs(120)).min(full)
5505    } else {
5506        full
5507    }
5508}
5509
5510/// Run a wave and parse each reply, re-asking the seats whose reply was
5511/// unusable.
5512///
5513/// The re-ask is a nudge rather than the whole prompt again when the seat still
5514/// holds its conversation, which is the difference between a cheap retry and
5515/// paying for the entire candidate set twice.
5516///
5517/// A seat that hits a rate limit is **not** re-asked: the same call will fail
5518/// the same way until the limit resets, so spending a retry attempt on it is
5519/// pure waste. Its loss is recorded in `losses` and it is returned as a failure
5520/// like any other absent seat — the caller decides whether the panel still has
5521/// a quorum.
5522#[allow(clippy::too_many_arguments)]
5523async fn ask_json_wave<T>(
5524    jobs: Vec<SeatJob>,
5525    sem: Arc<Semaphore>,
5526    retries: usize,
5527    ctx: &WaveCtx<'_>,
5528    losses: &mut Vec<QuotaLoss>,
5529    state: &mut RunState,
5530    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
5531) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
5532where
5533    T: serde::de::DeserializeOwned + Send + 'static,
5534{
5535    let n = jobs.len();
5536    let originals: Vec<SeatJob> = jobs;
5537    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
5538    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
5539    // Which attempt each seat's `done[i]` reflects — 0 for a first-ask
5540    // answer, N once it has gone through N nudges. Read back once this
5541    // returns, so a caller building a history record (`ReviewRecord`) can
5542    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
5543    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
5544    // field's own doc.
5545    let mut attempts_used: Vec<usize> = vec![0; n];
5546    let mut pending: Vec<usize> = (0..n).collect();
5547
5548    for attempt in 0..=retries {
5549        if pending.is_empty() {
5550            break;
5551        }
5552        let mut batch = Vec::with_capacity(pending.len());
5553        for &i in &pending {
5554            let src = &originals[i];
5555            // The prompt and the budget are one decision: a nudge restates
5556            // finished work, a re-sent prompt redoes it.
5557            let (prompt, timeout) = if attempt == 0 {
5558                (src.prompt.clone(), src.timeout)
5559            } else {
5560                let why = done[i]
5561                    .as_ref()
5562                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
5563                    .unwrap_or_else(|| "no parsable answer".to_owned());
5564                let nudge = prompt::nudge(&why);
5565                let nudged = has_context(&src.spec, &seats[i], src.sessions);
5566                let prompt = if nudged {
5567                    nudge
5568                } else {
5569                    format!("{}\n\n---\n\n{}", src.prompt, nudge)
5570                };
5571                (prompt, retry_budget(src.timeout, nudged))
5572            };
5573            batch.push(SeatJob {
5574                spec: src.spec.clone(),
5575                seat: seats[i].clone(),
5576                cwd: src.cwd.clone(),
5577                prompt,
5578                timeout,
5579                allow_write: src.allow_write,
5580                sessions: src.sessions,
5581                artifacts: src.artifacts.clone(),
5582                stem: if attempt == 0 {
5583                    src.stem.clone()
5584                } else {
5585                    format!("{}-retry{attempt}", src.stem)
5586                },
5587            });
5588        }
5589
5590        if attempt > 0 {
5591            let seats_out: Vec<&str> = pending
5592                .iter()
5593                .map(|&i| originals[i].seat.key.as_str())
5594                .collect();
5595            state.event(
5596                ctx.node,
5597                format!("retry {attempt}: re-asking {}", seats_out.join(", ")),
5598            );
5599        }
5600        let results = wave(batch, Arc::clone(&sem), ctx, state, attempt).await;
5601        let mut still = Vec::new();
5602        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
5603            seats[i] = seat;
5604            let (parsed, quota) = match out {
5605                AgentOutcome::Ok(o) => (
5606                    match verdict::extract_json::<T>(&o.text) {
5607                        Ok(v) => match validate(&v) {
5608                            Ok(()) => Ok((v, o)),
5609                            Err(e) => Err(e),
5610                        },
5611                        Err(e) => Err(e),
5612                    },
5613                    false,
5614                ),
5615                AgentOutcome::Quota(o) => {
5616                    losses.push(QuotaLoss {
5617                        seat: originals[i].seat.key.clone(),
5618                        node: ctx.node.to_owned(),
5619                        at: Timestamp::now(),
5620                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5621                    });
5622                    (
5623                        Err(anyhow::anyhow!("rate limited (quota); not retrying now")),
5624                        true,
5625                    )
5626                }
5627                // Not a parseable answer, but also not worth a special-cased
5628                // retry here: the nudge loop above already re-asks anything
5629                // that fails to parse, which is exactly what a dropped stream
5630                // needs. Just don't hand its raw error JSON to `extract_json`.
5631                AgentOutcome::Dropped(o) => {
5632                    let why = o
5633                        .dropped
5634                        .as_ref()
5635                        .map(|d| d.why.as_str())
5636                        .unwrap_or("the CLI ended the stream without delivering its answer");
5637                    (
5638                        Err(anyhow::anyhow!("the CLI dropped the stream ({why})")),
5639                        false,
5640                    )
5641                }
5642                AgentOutcome::Failed(e) => (Err(anyhow::anyhow!(e)), false),
5643            };
5644            let failed = parsed.is_err();
5645            done[i] = Some(parsed);
5646            attempts_used[i] = attempt;
5647            // Do not re-ask a rate-limited seat (quota) — a retry is known to
5648            // fail the same way; and never re-ask a seat that already parsed.
5649            if failed && !quota {
5650                still.push(i);
5651            }
5652        }
5653        pending = still;
5654    }
5655
5656    seats
5657        .into_iter()
5658        .zip(done)
5659        .zip(attempts_used)
5660        .map(|((seat, res), attempts)| {
5661            (
5662                seat,
5663                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
5664                attempts,
5665            )
5666        })
5667        .collect()
5668}
5669
5670/// Acquire the shared build cache's lease, waiting out contention within
5671/// `budget` (never past it — see AGENTS.md's build-cache section on why an
5672/// unbounded wait is never acceptable).
5673///
5674/// A first, non-blocking check happens before ever waiting; if it finds the
5675/// lease busy, that fact is logged as a `verify` event *and* flushed with
5676/// [`RunState::save`] immediately — not only once the wait finally succeeds
5677/// or gives up — so a `magi show` run by a different process while this one
5678/// is still waiting reads a `run.json` that says so, rather than whatever it
5679/// looked like before the wait started. The same applies to the terminal
5680/// failure: logged and saved before this returns `Err`, so a caller that
5681/// could not get the lease at all still leaves a legible record of why.
5682async fn acquire_cache_lease(
5683    state: &mut RunState,
5684    cache_dir: &Path,
5685    owner: &crate::cache::Owner,
5686    budget: Duration,
5687    context: &str,
5688) -> Result<crate::cache::Guard> {
5689    let home = crate::run::home();
5690    let started = Instant::now();
5691    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
5692        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(g),
5693        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
5694        Err(e) => {
5695            state.event(
5696                "verify",
5697                format!("{context}: could not check the shared build cache: {e:#}"),
5698            );
5699            if let Err(e2) = state.save() {
5700                tracing::warn!("could not persist a cache-check failure: {e2:#}");
5701            }
5702            return Err(e);
5703        }
5704    };
5705    state.event(
5706        "verify",
5707        format!(
5708            "{context}: waiting for the shared build cache at {} ({})",
5709            cache_dir.display(),
5710            busy.describe()
5711        ),
5712    );
5713    if let Err(e) = state.save() {
5714        tracing::warn!("could not persist a cache wait: {e:#}");
5715    }
5716    let remaining = budget.saturating_sub(started.elapsed());
5717    match crate::cache::wait_for(&home, cache_dir, owner, remaining, Duration::from_secs(5)).await {
5718        Ok(g) => Ok(g),
5719        Err(e) => {
5720            state.event("verify", format!("{context}: {e:#}"));
5721            if let Err(e2) = state.save() {
5722                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
5723            }
5724            Err(e)
5725        }
5726    }
5727}
5728
5729/// Run `body` — a verify command batch — while holding the shared build
5730/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
5731/// never interleave with another borrower's build against the same
5732/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
5733/// timeout, or a human's own `magi review`. See the `cache` module doc for
5734/// why this matters more than Cargo's own per-target locking covers — two
5735/// *different* worktrees building the same package name/version into one
5736/// cache directory is a staleness bug, not a lock contention one.
5737///
5738/// The wait for the lease is carved out of `budget`, never on top of it —
5739/// `body` is handed whatever is left, so a caller's own node timeout is the
5740/// only clock involved, exactly what AGENTS.md's build-cache section asks
5741/// for ("never an unbounded wait"). When `cache_dir` is `None` — no shared
5742/// cache configured at all — this is a pass-through: `body` runs with the
5743/// full budget and nothing is leased.
5744///
5745/// A lease that cannot be acquired within `budget` is reported as a single
5746/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
5747/// verification — the same shape a spawn failure already takes in
5748/// [`run_commands`], so a caller need not special-case it.
5749#[allow(clippy::too_many_arguments)]
5750async fn with_cache_lease<'s, F, Fut>(
5751    state: &'s mut RunState,
5752    cache_dir: Option<&Path>,
5753    node: &str,
5754    seat: &str,
5755    worktree: &Path,
5756    head: &str,
5757    budget: Duration,
5758    context: &str,
5759    body: F,
5760) -> (Vec<CommandOutcome>, bool)
5761where
5762    F: FnOnce(&'s mut RunState, Duration) -> Fut,
5763    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
5764{
5765    let Some(cache_dir) = cache_dir else {
5766        let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
5767        return (outcomes, retried);
5768    };
5769    let home = crate::run::home();
5770    let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
5771    let started = Instant::now();
5772    let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context).await {
5773        Ok(g) => g,
5774        Err(e) => {
5775            return (
5776                vec![CommandOutcome {
5777                    command: "(waiting for the shared build cache)".to_owned(),
5778                    code: None,
5779                    output_tail: e.to_string(),
5780                    duration_ms: started.elapsed().as_millis() as u64,
5781                    resource_blocked: true,
5782                }],
5783                false,
5784            );
5785        }
5786    };
5787    let identity = crate::cache::Identity::new(worktree, head);
5788    if let Err(e) = crate::cache::ensure_fresh(&home, cache_dir, &identity) {
5789        // A failed freshness check means this process cannot vouch for what
5790        // is sitting in the cache right now - on Windows this is exactly the
5791        // "a stale test executable is still locked, `cargo clean -p` cannot
5792        // remove it" case the evidence log records. Running verify anyway
5793        // and reporting whatever it says would let a result nobody can trust
5794        // stand for the tree it claims to have checked; fail the step
5795        // instead of the patch.
5796        state.event(
5797            "verify",
5798            format!(
5799                "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
5800                worktree.display(),
5801                short(head)
5802            ),
5803        );
5804        guard.release();
5805        return (
5806            vec![CommandOutcome {
5807                command: "(confirming the shared build cache is fresh)".to_owned(),
5808                code: None,
5809                output_tail: e.to_string(),
5810                duration_ms: started.elapsed().as_millis() as u64,
5811                resource_blocked: true,
5812            }],
5813            false,
5814        );
5815    }
5816    let remaining = budget.saturating_sub(started.elapsed());
5817    let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
5818    // A timed-out command's process was only *asked* to die (`kill_on_drop`,
5819    // `start_kill`); confirm it actually has before handing the directory to
5820    // the next acquirer. See `wait_for_timed_out_children_to_die`'s own doc
5821    // for what this can and cannot see.
5822    if !timed_out_pids.is_empty() {
5823        wait_for_timed_out_children_to_die(&timed_out_pids).await;
5824    }
5825    guard.release();
5826    (outcomes, retried)
5827}
5828
5829/// Poll `pids` — commands [`run_commands`] reports as still running when its
5830/// own timeout elapsed — until every one is confirmed gone, or
5831/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
5832///
5833/// Real confirmation where confirmation is possible, not a substitute for
5834/// full process-tree observation: a grandchild the timed-out process spawned
5835/// and that survives independently of it is invisible to a pid check the
5836/// same way it always was, and continuing to observe and collect *that*
5837/// stays a different piece of work with its own owner. This only narrows a
5838/// fixed blind wait into an actual check of the pids this process does know
5839/// about.
5840async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
5841    wait_for_pids_with(
5842        pids,
5843        crate::proc::pid_alive,
5844        LEASE_RELEASE_POLL,
5845        LEASE_RELEASE_MAX_WAIT,
5846    )
5847    .await;
5848}
5849
5850/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
5851/// interval and ceiling supplied by the caller, so the polling *logic* -
5852/// returns as soon as every pid reports dead, gives up at the ceiling
5853/// otherwise - is testable on millisecond durations without asking the real
5854/// OS about a pid at all.
5855async fn wait_for_pids_with<F: Fn(u32) -> bool>(
5856    pids: &[u32],
5857    alive: F,
5858    poll: Duration,
5859    max_wait: Duration,
5860) {
5861    let deadline = Instant::now() + max_wait;
5862    loop {
5863        if pids.iter().all(|&pid| !alive(pid)) {
5864            return;
5865        }
5866        if Instant::now() >= deadline {
5867            return;
5868        }
5869        tokio::time::sleep(poll).await;
5870    }
5871}
5872
5873/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
5874/// admission that it could not even get a verify command to run, as opposed
5875/// to evidence the command actually produced? A caller that would otherwise
5876/// read a resource-blocked outcome as a red command must check this first:
5877/// see [`Runner::gate`], which retries rather than records `Blocked` when
5878/// this is true.
5879fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
5880    outcomes.iter().any(|o| o.resource_blocked)
5881}
5882
5883/// Describe one verify command's outcome for the event log, distinguishing a
5884/// build/link failure — the toolchain never produced a binary to run — from
5885/// an actual test failure, since only the latter is a verdict on the patch.
5886fn e2e_outcome_label(o: &CommandOutcome) -> String {
5887    if o.ok() {
5888        return "pass".to_owned();
5889    }
5890    let reason = if o.build_failed() {
5891        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
5892    } else {
5893        format!("FAIL ({:?})", o.code)
5894    };
5895    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
5896}
5897
5898/// Run `verify.e2e`, retrying once if the first attempt could not build or
5899/// link — a build/link failure is frequently a race against a shared
5900/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
5901/// `verify` event per command, tagged with `context` (normally `"round N"`)
5902/// so the two call sites that need this — the ordinary per-round leg in
5903/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
5904/// it will ever call a round green — read identically in the event log.
5905async fn run_e2e_with_retry(
5906    state: &mut RunState,
5907    shell: &[String],
5908    commands: &[String],
5909    worktree: &Path,
5910    timeout: Duration,
5911    context: &str,
5912) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
5913    let (mut e2e, mut timed_out_pids) = run_commands(
5914        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
5915    )
5916    .await;
5917    for o in &e2e {
5918        state.event(
5919            "verify",
5920            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
5921        );
5922    }
5923    // A build/link failure is not a verdict on the patch — it is frequently a
5924    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
5925    // one retry before letting a red like that decide the round.
5926    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
5927    if verify_retried {
5928        state.event(
5929            "verify",
5930            format!(
5931                "{context}: verify could not build/link, not a test result — retrying once \
5932                 before concluding"
5933            ),
5934        );
5935        let retried = run_commands(
5936            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
5937        )
5938        .await;
5939        e2e = retried.0;
5940        // Both attempts' timeouts matter, not just the last one: the first
5941        // attempt's descendants may still be alive alongside the retry's.
5942        timed_out_pids.extend(retried.1);
5943        for o in &e2e {
5944            state.event(
5945                "verify",
5946                format!(
5947                    "{context}: retry `{}` -> {}",
5948                    o.command,
5949                    e2e_outcome_label(o)
5950                ),
5951            );
5952        }
5953    }
5954    (e2e, verify_retried, timed_out_pids)
5955}
5956
5957/// Run configured shell commands in `cwd`, in order. The second element is
5958/// the pid of every command that hit `timeout` and was still running when
5959/// this stopped waiting on it (best-effort: `None` when the platform did not
5960/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
5961/// that releases a shared resource afterward needs to know.
5962///
5963/// Records `task` into [`RunState::active`] at every command boundary
5964/// (`RunState::task_command`) and clears it once the whole list has run
5965/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
5966/// for minutes with no seat and no output of its own to show for it (see
5967/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
5968/// apart from "not yet run" without this), and this is the only place that
5969/// knows which command is running right now and how many are left. Three
5970/// saves per command — start, not per second — matching the same "only at a
5971/// boundary" rule [`wave`] already follows for seats.
5972#[allow(clippy::too_many_arguments)]
5973async fn run_commands(
5974    state: &mut RunState,
5975    node: &str,
5976    task: &str,
5977    attempt: usize,
5978    shell: &[String],
5979    commands: &[String],
5980    cwd: &Path,
5981    timeout: Duration,
5982) -> (Vec<CommandOutcome>, Vec<u32>) {
5983    if commands.is_empty() {
5984        // Nothing to mark as running and nothing to clear — an empty list
5985        // means "not configured", and touching `active` (or the disk) over
5986        // that would be a write for every round of a repo with no
5987        // `verify.e2e` / `verify.gate` commands at all.
5988        return (Vec::new(), Vec::new());
5989    }
5990    let mut out = Vec::new();
5991    let mut timed_out_pids = Vec::new();
5992    let total = commands.len();
5993    for (idx, command) in commands.iter().enumerate() {
5994        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
5995        if let Err(e) = state.save() {
5996            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
5997        }
5998        let started = Instant::now();
5999        let mut cmd = tokio::process::Command::new(&shell[0]);
6000        cmd.quiet();
6001        cmd.args(&shell[1..])
6002            .arg(command)
6003            .current_dir(cwd)
6004            .stdin(std::process::Stdio::null())
6005            .stdout(std::process::Stdio::piped())
6006            .stderr(std::process::Stdio::piped())
6007            .kill_on_drop(true);
6008        let spawned = cmd.spawn();
6009        let (code, body) = match spawned {
6010            Ok(child) => {
6011                // Captured before the child is consumed below: `kill_on_drop`
6012                // only *asks* the process to die when the timeout branch
6013                // drops it, and the pid is the only way anyone downstream can
6014                // later check whether that request actually took.
6015                let pid = child.id();
6016                match tokio::time::timeout(timeout, child.wait_with_output()).await {
6017                    Ok(Ok(o)) => {
6018                        let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
6019                        body.push_str(&String::from_utf8_lossy(&o.stderr));
6020                        (o.status.code(), body)
6021                    }
6022                    Ok(Err(e)) => (None, format!("failed to run: {e}")),
6023                    Err(_) => {
6024                        if let Some(pid) = pid {
6025                            timed_out_pids.push(pid);
6026                        }
6027                        (None, format!("timed out after {}s", timeout.as_secs()))
6028                    }
6029                }
6030            }
6031            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
6032        };
6033        out.push(CommandOutcome {
6034            command: command.clone(),
6035            code,
6036            output_tail: tail(&body, OUTPUT_TAIL),
6037            duration_ms: started.elapsed().as_millis() as u64,
6038            resource_blocked: false,
6039        });
6040    }
6041    state.task_finished(task);
6042    if let Err(e) = state.save() {
6043        tracing::warn!("could not persist the end of {task}: {e:#}");
6044    }
6045    (out, timed_out_pids)
6046}
6047
6048/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
6049/// section (`report::run`) and in the `merge` event this node records — for
6050/// the operator to run by hand.
6051///
6052/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
6053/// branch whose ruleset forbids merge commits (GitHub's "must not contain
6054/// merge commits", or "require linear history") rejects the push a `--no-ff`
6055/// merge would produce, which is exactly the guidance this function replaces.
6056/// `message`'s first line becomes the squash commit's subject, matching the
6057/// note `report::run` prints alongside this command — see that function for
6058/// why an explicit subject is not optional there.
6059fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
6060    let repo = repo.display();
6061    match style {
6062        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
6063        MergeStyle::Squash => {
6064            let subject = message.lines().next().unwrap_or(branch);
6065            format!(
6066                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
6067            )
6068        }
6069        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
6070    }
6071}
6072
6073/// The merge commit / pull request body: the task, and — when the winning
6074/// review round was not clean — the findings still open and whatever the
6075/// fixer declined, so `merge = "pr"` hands the reader the same material
6076/// `magi show` does rather than a pull request that reads clean while
6077/// `run.json` disagrees.
6078///
6079/// The first line doubles as the squash/merge commit subject
6080/// (`manual_merge_command`), which takes it via `message.lines().next()`
6081/// verbatim — so it has to be the task's own opening line, not run/candidate
6082/// bookkeeping. The pull request title (`gh_pr_create`) starts from the same
6083/// line but is further reshaped and truncated by `pr_title` to stay inside
6084/// GitHub's limit; see that function for why. "Merge magi run ec12 (candidate
6085/// B)" told a reader nothing about what landed once the run id had scrolled
6086/// off the PR list. That bookkeeping still needs to be findable, just not
6087/// from the title: the branch name already carries it
6088/// (`RunState::branch_for`), and the footer below repeats it as plain tags
6089/// for a reader holding only the merged commit or the PR body.
6090///
6091/// `state.instruction` can open with blank lines — a `--file` task is passed
6092/// through verbatim (`task_text` only rejects a body that is blank
6093/// *entirely*) — and `.lines().next()` on those reads back as `Some("")`, not
6094/// `None`. `trim_start` drops exactly those leading blank lines so the first
6095/// line is the task's real opening line, and the empty-after-trim case (a
6096/// whitespace-only instruction) falls back the same way `queue::title_from`
6097/// does for the same situation.
6098fn pr_body(state: &RunState, winner: char) -> String {
6099    let instruction = state.instruction.trim_start();
6100    let mut message = if instruction.is_empty() {
6101        "(empty task)".to_owned()
6102    } else {
6103        instruction.to_owned()
6104    };
6105
6106    let open = state.open_findings();
6107    if !open.is_empty() {
6108        message.push_str("\n\n## Open review findings\n\n");
6109        for f in &open {
6110            message.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
6111        }
6112    }
6113
6114    if let Some(fix) = state.reviews.last().and_then(|r| r.fix.as_ref())
6115        && !fix.rejected.is_empty()
6116    {
6117        message.push_str("\n## Declined by the fixer\n\n");
6118        for r in &fix.rejected {
6119            message.push_str(&format!("- `{}`: {}\n", r.id, r.why));
6120        }
6121    }
6122
6123    message.push_str(&format!(
6124        "\n\n---\nmagi:run/{} magi:candidate-{}\n",
6125        state.id,
6126        winner.to_ascii_lowercase()
6127    ));
6128
6129    message
6130}
6131
6132/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
6133/// under the hood, rejects a `title` over 256 characters and the whole
6134/// command fails — no PR at all, for a run whose body was otherwise fine
6135/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
6136/// that limit: `title_from` counts `chars()` (Unicode scalars), which is not
6137/// always how GitHub counts, plus one character for the trailing ellipsis
6138/// `title_from` may add. It is a margin, not a guarantee — a title packed
6139/// with multi-unit characters could still in principle land close to the
6140/// edge, but a real task title's occasional emoji or accented letter fits
6141/// comfortably inside it.
6142const PR_TITLE_MAX: usize = 240;
6143
6144/// The pull request title: the PR body's first line, reshaped and truncated
6145/// by [`queue::title_from`] the same way `magi show`'s task list titles are,
6146/// so it stays inside GitHub's limit on `--title` (see [`PR_TITLE_MAX`]).
6147fn pr_title(body: &str) -> String {
6148    queue::title_from(body, PR_TITLE_MAX)
6149}
6150
6151/// `gh pr create`, returning the PR url.
6152async fn gh_pr_create(cwd: &Path, base: &str, head: &str, body: &str) -> Result<String> {
6153    let title = pr_title(body);
6154    let out = tokio::process::Command::new("gh")
6155        .args([
6156            "pr", "create", "--base", base, "--head", head, "--title", &title, "--body", body,
6157        ])
6158        .current_dir(cwd)
6159        .quiet()
6160        .stdin(std::process::Stdio::null())
6161        .output()
6162        .await
6163        .context("spawn gh")?;
6164    if out.status.success() {
6165        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
6166    } else {
6167        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
6168    }
6169}
6170
6171/// Tear a run's worktrees and branches down.
6172///
6173/// `home` is where the updated `run.json` is saved (via
6174/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
6175/// a housekeeping pass already has its own honest `home` handed to it, and
6176/// falling through to the global here would write back through whichever
6177/// directory some other process or test pinned into that `OnceLock` first,
6178/// not the one the caller actually resolved its `runs` and `state` from.
6179pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
6180    let repo = state.repo.clone();
6181    let root = state.worktree_root();
6182    let winner = state.tally.as_ref().map(|t| t.winner);
6183    let mut removed = Vec::new();
6184
6185    for i in 0..state.candidates.len() {
6186        let c = state.candidates[i].clone();
6187        let is_winner = Some(c.label) == winner;
6188        if is_winner && !drop_winner {
6189            continue;
6190        }
6191        if c.worktree.exists() {
6192            git::worktree_remove(&repo, &c.worktree).await.ok();
6193            removed.push(c.worktree.to_string_lossy().into_owned());
6194        }
6195        if git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
6196            git::branch_delete(&repo, &c.branch).await.ok();
6197            removed.push(c.branch.clone());
6198        }
6199        state.candidates[i].folded = true;
6200    }
6201
6202    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
6203        let path = name.path();
6204        let keep = !drop_winner
6205            && winner.is_some_and(|w| {
6206                path.file_name()
6207                    .is_some_and(|n| n == format!("cand-{w}").as_str())
6208            });
6209        if keep {
6210            continue;
6211        }
6212        git::worktree_remove(&repo, &path).await.ok();
6213        removed.push(path.to_string_lossy().into_owned());
6214    }
6215
6216    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
6217    // judge worktrees, so once the loop above has cleared all of them out,
6218    // the parent is a bare directory nobody else was ever going to remove -
6219    // git only ever managed what was inside it. Left alone, one of these
6220    // accumulates per fully-folded run; the operator's own machine had 74.
6221    // `remove_if_empty` re-checks rather than assuming: a run whose winner
6222    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
6223    // so does anything a run never claimed that happens to share the bay.
6224    remove_if_empty(&root);
6225
6226    if state.enabled_worktree_config && drop_winner {
6227        // A release, not a raw disable: some sibling run in this repository
6228        // may still hold its own reference (see `git::acquire_worktree_config`),
6229        // and only the last release actually turns the setting back off.
6230        git::release_worktree_config(&repo).await.ok();
6231        state.enabled_worktree_config = false;
6232    }
6233    state.save_under(home)?;
6234    Ok(removed)
6235}
6236
6237/// Remove `dir` if it exists and has nothing in it.
6238///
6239/// Best-effort and silent by design: a directory that is not empty (a run
6240/// whose winner is still parked there, a stray file some other process left)
6241/// is exactly the case this must refuse, and a directory that is already gone
6242/// is not a failure worth reporting either. `std::fs::remove_dir` itself
6243/// already refuses a non-empty directory, so the emptiness check below is
6244/// belt, not suspenders - it is what keeps this from ever attempting the
6245/// removal in the case that matters, rather than trusting `remove_dir`'s
6246/// error path to have no side effects if it ever changed.
6247fn remove_if_empty(dir: &Path) {
6248    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
6249        std::fs::remove_dir(dir).ok();
6250    }
6251}
6252
6253/// Severity of the worst open finding in the last review round, for reporting.
6254pub fn worst_open(state: &RunState) -> Option<Severity> {
6255    state
6256        .reviews
6257        .last()?
6258        .reviews
6259        .iter()
6260        .flat_map(|r| r.findings.iter())
6261        .map(|f| f.severity)
6262        .max()
6263}
6264
6265#[cfg(test)]
6266mod tests {
6267    use super::*;
6268    use crate::run::GateStatus;
6269    use std::collections::BTreeMap;
6270    use std::time::Duration;
6271
6272    fn conductor() -> AgentSpec {
6273        AgentSpec {
6274            id: "conductor".to_owned(),
6275            kind: crate::config::AgentKind::Command,
6276            model: None,
6277            command: vec!["true".to_owned()],
6278            extra_args: Vec::new(),
6279            env: BTreeMap::new(),
6280            prompt_delivery: None,
6281        }
6282    }
6283
6284    fn spec(id: &str) -> AgentSpec {
6285        AgentSpec {
6286            id: id.to_owned(),
6287            kind: crate::config::AgentKind::Command,
6288            model: None,
6289            command: vec!["true".to_owned()],
6290            extra_args: Vec::new(),
6291            env: BTreeMap::new(),
6292            prompt_delivery: None,
6293        }
6294    }
6295
6296    // `next_untried_implementer` is the property `resume_quota_losses`'s own
6297    // fallback loop depends on to terminate: it must walk forward from the
6298    // seat's own position, never restart at the front of the roster, and it
6299    // must never hand back an id already tried, however many times that id
6300    // happens to appear.
6301
6302    #[test]
6303    fn next_untried_implementer_walks_forward_from_the_seats_own_position() {
6304        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
6305        let tried = BTreeSet::from(["beta".to_owned()]);
6306        // beta sits at index 1; the next candidate is gamma, never alpha —
6307        // which is very likely a different candidate slot's own agent.
6308        let next = next_untried_implementer(&roster, 1, &tried);
6309        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
6310    }
6311
6312    #[test]
6313    fn next_untried_implementer_does_not_wrap_back_past_its_own_start() {
6314        let roster = vec![spec("alpha"), spec("beta")];
6315        let tried = BTreeSet::from(["beta".to_owned()]);
6316        // beta is the roster's last entry: nothing follows it, and alpha —
6317        // earlier in the roster, almost certainly a different candidate
6318        // slot's own agent — must not be reached by wrapping back to it.
6319        assert!(next_untried_implementer(&roster, 1, &tried).is_none());
6320    }
6321
6322    #[test]
6323    fn next_untried_implementer_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
6324        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
6325        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
6326        // beta (index 1) and gamma (index 2, the only entry after it) have
6327        // both been tried; alpha (index 0) never has, but it comes before
6328        // beta's own position, so there is nothing further for this seat.
6329        assert!(next_untried_implementer(&roster, 1, &tried).is_none());
6330    }
6331
6332    #[test]
6333    fn next_untried_implementer_skips_ids_already_tried_even_when_duplicated() {
6334        let roster = vec![spec("a"), spec("a"), spec("b")];
6335        let tried = BTreeSet::from(["a".to_owned()]);
6336        let next = next_untried_implementer(&roster, 0, &tried);
6337        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
6338    }
6339
6340    #[test]
6341    fn next_untried_implementer_returns_none_once_every_id_is_tried() {
6342        let roster = vec![spec("a"), spec("b")];
6343        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
6344        assert!(next_untried_implementer(&roster, 0, &tried).is_none());
6345    }
6346
6347    #[test]
6348    fn remove_if_empty_only_ever_takes_a_bare_directory() {
6349        let dir = tempfile::tempdir().unwrap();
6350        let bay = dir.path().join("ffff");
6351
6352        // Not there yet: nothing to do, nothing to panic on.
6353        remove_if_empty(&bay);
6354        assert!(!bay.exists());
6355
6356        // Something still inside - the winner's worktree, or a stray file -
6357        // keeps the directory standing.
6358        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
6359        remove_if_empty(&bay);
6360        assert!(bay.exists(), "non-empty directory must survive");
6361
6362        // Once the last entry is gone, so is the directory itself.
6363        std::fs::remove_dir(bay.join("cand-A")).unwrap();
6364        remove_if_empty(&bay);
6365        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
6366    }
6367
6368    // `round_is_clean` is the exact decision this task fixed: a round with a
6369    // seat that never answered must not read the same as a round every seat
6370    // actually reviewed. These are deterministic and process-free by design —
6371    // the equivalent end-to-end check (a real reviewer timing out under a
6372    // live graph run) is a genuine race against wall-clock contention, and a
6373    // spawn slow enough to blow even a generous budget under a loaded test
6374    // run must not turn this specific regression check flaky.
6375
6376    #[test]
6377    fn a_full_panel_that_found_nothing_is_clean() {
6378        assert!(round_is_clean(
6379            0,
6380            true,
6381            2,
6382            2,
6383            0,
6384            IncompleteReviewPolicy::Block
6385        ));
6386    }
6387
6388    #[test]
6389    fn a_missing_seat_is_never_clean_under_the_default_policy() {
6390        assert!(!round_is_clean(
6391            0,
6392            true,
6393            1,
6394            2,
6395            0,
6396            IncompleteReviewPolicy::Block
6397        ));
6398    }
6399
6400    #[test]
6401    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
6402        assert!(!round_is_clean(
6403            1,
6404            true,
6405            1,
6406            2,
6407            0,
6408            IncompleteReviewPolicy::Warn
6409        ));
6410    }
6411
6412    #[test]
6413    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
6414        assert!(round_is_clean(
6415            0,
6416            true,
6417            1,
6418            2,
6419            0,
6420            IncompleteReviewPolicy::Warn
6421        ));
6422    }
6423
6424    #[test]
6425    fn a_full_panel_with_an_open_finding_is_not_clean() {
6426        assert!(!round_is_clean(
6427            1,
6428            true,
6429            2,
6430            2,
6431            0,
6432            IncompleteReviewPolicy::Block
6433        ));
6434    }
6435
6436    #[test]
6437    fn a_full_panel_with_a_red_e2e_is_not_clean() {
6438        assert!(!round_is_clean(
6439            0,
6440            false,
6441            2,
6442            2,
6443            0,
6444            IncompleteReviewPolicy::Block
6445        ));
6446    }
6447
6448    // The stall this task closes: under the default `block` policy, a seat
6449    // missing only because it was rate limited must not force a wait for a
6450    // session limit that will not lift by the next round. `round_is_clean`
6451    // is where that quorum carve-out lives; the review loop around it never
6452    // changes what a reviewer's vote or a finding's severity means.
6453
6454    #[test]
6455    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
6456        // 1 of 2 answered, and the one missing was quota'd — the exact
6457        // "review-2 rate limited (quota)" shape from the field report.
6458        assert!(round_is_clean(
6459            0,
6460            true,
6461            1,
6462            2,
6463            1,
6464            IncompleteReviewPolicy::Block
6465        ));
6466    }
6467
6468    #[test]
6469    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
6470        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
6471        // not a quota loss (`quota_missing` stays 0) — worth another try.
6472        assert!(!round_is_clean(
6473            0,
6474            true,
6475            1,
6476            2,
6477            0,
6478            IncompleteReviewPolicy::Block
6479        ));
6480    }
6481
6482    #[test]
6483    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
6484        assert!(!round_is_clean(
6485            1,
6486            true,
6487            1,
6488            2,
6489            1,
6490            IncompleteReviewPolicy::Block
6491        ));
6492        assert!(!round_is_clean(
6493            0,
6494            false,
6495            1,
6496            2,
6497            1,
6498            IncompleteReviewPolicy::Block
6499        ));
6500    }
6501
6502    #[test]
6503    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
6504        // Every seat quota'd, nobody answered: there is no panel to decide
6505        // on, so this must fall through to the existing block-and-retry
6506        // fallback rather than call an unreviewed patch clean.
6507        assert!(!round_is_clean(
6508            0,
6509            true,
6510            0,
6511            2,
6512            2,
6513            IncompleteReviewPolicy::Block
6514        ));
6515    }
6516
6517    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
6518        CommandOutcome {
6519            command: "test".to_owned(),
6520            code,
6521            output_tail: String::new(),
6522            duration_ms: 0,
6523            resource_blocked,
6524        }
6525    }
6526
6527    #[test]
6528    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
6529        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
6530        assert!(
6531            !verify_inconclusive(&[outcome(Some(1), false)]),
6532            "an ordinary failure is still evidence about the patch"
6533        );
6534        assert!(verify_inconclusive(&[outcome(None, true)]));
6535        assert!(
6536            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
6537            "one inconclusive outcome taints the whole batch"
6538        );
6539        assert!(!verify_inconclusive(&[]));
6540    }
6541
6542    #[tokio::test]
6543    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
6544        // Alive for the first two checks, then dead - confirms the loop
6545        // actually re-polls rather than deciding once and sleeping out the
6546        // ceiling regardless.
6547        let calls = std::sync::atomic::AtomicUsize::new(0);
6548        let started = Instant::now();
6549        wait_for_pids_with(
6550            &[123],
6551            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
6552            Duration::from_millis(5),
6553            Duration::from_secs(5),
6554        )
6555        .await;
6556        assert!(
6557            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
6558            "must keep checking rather than deciding on the first answer"
6559        );
6560        assert!(
6561            started.elapsed() < Duration::from_secs(1),
6562            "must return the moment it is confirmed dead, not wait out the ceiling"
6563        );
6564    }
6565
6566    #[tokio::test]
6567    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
6568        let started = Instant::now();
6569        wait_for_pids_with(
6570            &[123],
6571            |_| true, // never reports dead
6572            Duration::from_millis(5),
6573            Duration::from_millis(30),
6574        )
6575        .await;
6576        let elapsed = started.elapsed();
6577        assert!(
6578            elapsed >= Duration::from_millis(30),
6579            "must not give up before its own ceiling: {elapsed:?}"
6580        );
6581        assert!(
6582            elapsed < Duration::from_secs(1),
6583            "must not wait past its own ceiling either: {elapsed:?}"
6584        );
6585    }
6586
6587    #[tokio::test]
6588    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
6589        let started = Instant::now();
6590        wait_for_pids_with(
6591            &[],
6592            |_| true,
6593            Duration::from_secs(5),
6594            Duration::from_secs(5),
6595        )
6596        .await;
6597        assert!(
6598            started.elapsed() < Duration::from_millis(200),
6599            "an empty pid list has nothing to confirm"
6600        );
6601    }
6602
6603    // `review_conclusion` is the exact decision the review hand-off task
6604    // fixed: a round budget spent (or a tree that stopped moving) must not
6605    // collapse into `Blocked` regardless of what verification actually
6606    // said. Deterministic and process-free for the same reason the
6607    // `round_is_clean` family above is.
6608    fn review_round(
6609        clean: bool,
6610        blocking: usize,
6611        answered: usize,
6612        expected: usize,
6613        progressed: bool,
6614        e2e_ok: bool,
6615    ) -> ReviewRound {
6616        ReviewRound {
6617            round: 1,
6618            head: "h".to_owned(),
6619            verified_head: None,
6620            verified_at: None,
6621            reviews: Vec::new(),
6622            e2e: vec![CommandOutcome {
6623                command: "test".to_owned(),
6624                code: Some(if e2e_ok { 0 } else { 1 }),
6625                output_tail: String::new(),
6626                duration_ms: 0,
6627                resource_blocked: false,
6628            }],
6629            verify_retried: false,
6630            e2e_deferred: false,
6631            e2e_defer_reason: None,
6632            fix: None,
6633            blocking,
6634            answered,
6635            expected,
6636            clean,
6637            progressed,
6638            vote_split: false,
6639            reconsideration: Vec::new(),
6640            verdict: None,
6641        }
6642    }
6643
6644    #[test]
6645    fn review_conclusion_is_none_when_nothing_has_run() {
6646        assert_eq!(review_conclusion(&[], 3), None);
6647    }
6648
6649    #[test]
6650    fn review_conclusion_is_none_while_rounds_remain() {
6651        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
6652        assert_eq!(review_conclusion(&rounds, 3), None);
6653    }
6654
6655    #[test]
6656    fn review_conclusion_is_gating_once_a_round_is_clean() {
6657        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
6658        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
6659    }
6660
6661    #[test]
6662    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
6663        let rounds = vec![
6664            review_round(false, 1, 2, 2, true, true),
6665            review_round(false, 1, 2, 2, true, true),
6666        ];
6667        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
6668    }
6669
6670    #[test]
6671    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
6672        let rounds = vec![
6673            review_round(false, 1, 2, 2, true, true),
6674            review_round(false, 1, 2, 2, true, false),
6675        ];
6676        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
6677    }
6678
6679    #[test]
6680    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
6681        // Magi never got a command to run against this round's own head — a
6682        // resource-blocked attempt, not a red one — so this must never
6683        // settle on `Blocked` the way a genuine e2e failure would. `None`
6684        // here is what tells `Runner::review_loop` to retry the check
6685        // itself rather than trust this cheap recomputation with a verdict
6686        // it cannot actually produce.
6687        let mut blocked = review_round(false, 1, 2, 2, true, false);
6688        blocked.e2e[0].resource_blocked = true;
6689        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
6690        assert_eq!(review_conclusion(&rounds, 2), None);
6691    }
6692
6693    #[test]
6694    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
6695        // Missing input, not a verified tree — never a hand-off candidate.
6696        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
6697        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
6698    }
6699
6700    #[test]
6701    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
6702        let rounds = vec![
6703            review_round(false, 1, 2, 2, false, true),
6704            review_round(false, 1, 2, 2, false, true),
6705        ];
6706        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
6707    }
6708
6709    fn secs(n: u64) -> Duration {
6710        Duration::from_secs(n)
6711    }
6712
6713    /// A throwaway repo with one commit on `main`, for tests that need `merge`
6714    /// to make real (and, if it runs at all, real*ly fail*) git calls.
6715    fn init_repo(dir: &Path) {
6716        let run = |args: &[&str]| {
6717            let out = std::process::Command::new("git")
6718                .args(args)
6719                .current_dir(dir)
6720                .quiet()
6721                .output()
6722                .expect("spawn git");
6723            assert!(
6724                out.status.success(),
6725                "git {args:?} failed: {}",
6726                String::from_utf8_lossy(&out.stderr)
6727            );
6728        };
6729        run(&["init", "-b", "main"]);
6730        run(&["config", "user.name", "magi test"]);
6731        run(&["config", "user.email", "magi@example.com"]);
6732        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
6733        run(&["add", "-A"]);
6734        run(&["commit", "-m", "init"]);
6735    }
6736
6737    // `settle_questions` is what closes the ghost the phone showed: a run's
6738    // seat asked something, the run then ended, and nothing was left to
6739    // abandon the question it left `open`. `HOME` is a process-wide
6740    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
6741    // first time it runs in the binary — every test below still reaches the
6742    // same directory whichever call won, and each gets its own run id from
6743    // `RunState::new`, so they never collide there.
6744    fn ask_test_home() {
6745        crate::run::set_home(std::env::temp_dir().join("magi-graph-ask-tests-home"));
6746    }
6747
6748    /// A minimal, git-free `Runner` at a given status — `settle_questions`
6749    /// reads nothing else off it.
6750    fn runner_at(status: RunStatus) -> Runner {
6751        let mut state = RunState::new(
6752            PathBuf::from("/nonexistent/repo"),
6753            "main".to_owned(),
6754            "deadbeef".to_owned(),
6755            "task".to_owned(),
6756            Config::default(),
6757        );
6758        state.status = status;
6759        Runner {
6760            state,
6761            roles: ResolvedRoles {
6762                implementers: Vec::new(),
6763                judges: Vec::new(),
6764                reviewers: Vec::new(),
6765                fixer: None,
6766                conductor: conductor(),
6767                implementer_roster: Vec::new(),
6768            },
6769            sem: Arc::new(Semaphore::new(1)),
6770            pause: Pause::new(),
6771            interrupt: Pause::new(),
6772        }
6773    }
6774
6775    /// `park_here` folding in the reason `Pause::park_because` recorded -
6776    /// this is what lets an operator reading a run's events tell an
6777    /// interrupt-driven park from an ordinary shutdown park.
6778    #[test]
6779    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
6780        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
6781        let mut runner = runner_at(RunStatus::Implementing);
6782        let interrupt = Pause::new();
6783        runner.watch_interrupt(interrupt.clone());
6784
6785        interrupt.park_because("task a1b2 asked to run first");
6786
6787        assert!(runner.park_here().expect("park_here"));
6788        assert!(runner.state.parked);
6789        let last = runner.state.events.last().expect("a park event");
6790        assert_eq!(last.node, "park");
6791        assert!(
6792            last.message.contains("task a1b2 asked to run first"),
6793            "expected the interrupt reason in {:?}",
6794            last.message
6795        );
6796    }
6797
6798    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
6799    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
6800    /// cleared) must not make a *different* run - one only watching its own,
6801    /// unshared interrupt `Pause` - see itself as parked. If a future change
6802    /// ever collapsed these back into one handle, the interrupt scheduler
6803    /// would park every run for the rest of the daemon's life, not just the
6804    /// one it meant to interrupt.
6805    #[test]
6806    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
6807        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
6808        let mut runner = runner_at(RunStatus::Implementing);
6809        let shutdown = Pause::new();
6810        runner.on_pause(shutdown.clone());
6811        let interrupt = Pause::new();
6812        runner.watch_interrupt(interrupt.clone());
6813
6814        // Nobody has asked for anything yet.
6815        assert!(!runner.park_here().expect("park_here"));
6816        assert!(!runner.state.parked);
6817
6818        // Only the interrupt handle fires; the shutdown handle stays clear.
6819        interrupt.park_because("test");
6820        assert!(!shutdown.parked());
6821        assert!(runner.park_here().expect("park_here"));
6822    }
6823
6824    /// The property every prior attempt at this feature failed to pin down:
6825    /// asking a run to park while one of its nodes has a real, in-flight
6826    /// async operation running (an agent call, in production) must not cut
6827    /// that operation short. `park_here` is only ever consulted *between*
6828    /// `execute`'s node calls - see its own doc - so nothing inside a node
6829    /// can observe a park request until the node itself returns. This proves
6830    /// that structurally, with real `tokio` concurrency and a channel
6831    /// handshake (never a sleep, which would only prove "usually", not
6832    /// "cannot"): the "node" below reports that it has genuinely started,
6833    /// and only then is the park requested; the node still has to be told to
6834    /// finish before `park_here` is ever called, exactly mirroring every
6835    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
6836    /// in `execute`.
6837    #[tokio::test]
6838    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
6839        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
6840        let mut runner = runner_at(RunStatus::Implementing);
6841        let interrupt = Pause::new();
6842        runner.watch_interrupt(interrupt.clone());
6843
6844        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
6845        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
6846
6847        // Stands in for one node's in-flight agent call: it proves it has
6848        // genuinely started, then blocks - exactly as a spawned CLI process
6849        // does - until told to finish.
6850        let node = async move {
6851            started_tx.send(()).expect("send started");
6852            finish_rx.await.expect("recv finish");
6853            "node finished"
6854        };
6855
6856        let interrupter = async move {
6857            started_rx.await.expect("recv started");
6858            // The call is now genuinely in flight. Ask it to park.
6859            interrupt.park_because("higher-priority task waiting");
6860            // Nothing the node does can observe this yet - there is no
6861            // check inside it, by construction - so let the executor run
6862            // anything pending and then let the node finish on its own.
6863            tokio::task::yield_now().await;
6864            finish_tx.send(()).expect("send finish");
6865        };
6866
6867        let (node_result, ()) = tokio::join!(node, interrupter);
6868        assert_eq!(
6869            node_result, "node finished",
6870            "the in-flight call ran to completion"
6871        );
6872
6873        // Only now, at the boundary the real `execute` would check right
6874        // after this node, does the park take effect.
6875        assert!(runner.park_here().expect("park_here"));
6876        assert!(runner.state.parked);
6877    }
6878
6879    /// A run parked mid-competition carries every field it had accumulated
6880    /// through the exact same disk round-trip an ordinary resume uses -
6881    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
6882    /// Nothing about parking for an interrupt is a special case of that path;
6883    /// this is what proves it rather than assuming it.
6884    #[test]
6885    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
6886        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
6887        let mut runner = runner_at(RunStatus::Judging);
6888        // `Runner::resume` re-resolves roles from the saved config, which
6889        // refuses an empty roster - give it the same minimal one `conductor`
6890        // itself uses.
6891        runner.state.config.agents = vec![conductor()];
6892        runner.state.candidates = vec![Candidate {
6893            index: 0,
6894            label: 'A',
6895            agent: "alpha".to_owned(),
6896            branch: "magi/x/A".to_owned(),
6897            worktree: PathBuf::from("/nonexistent/worktree"),
6898            summary: "did the thing".to_owned(),
6899            stat: "1 file changed".to_owned(),
6900            files: 1,
6901            commits: 1,
6902            empty: false,
6903            failed: None,
6904            verified_noop: None,
6905            duration_ms: 1234,
6906            folded: false,
6907        }];
6908        let run_id = runner.state.id.clone();
6909
6910        let interrupt = Pause::new();
6911        runner.watch_interrupt(interrupt.clone());
6912        interrupt.park_because("task c3d4 asked to run first");
6913        assert!(runner.park_here().expect("park_here"));
6914
6915        let resumed = Runner::resume(&run_id).expect("resume");
6916        assert_eq!(resumed.state.candidates.len(), 1);
6917        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
6918        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
6919        assert_eq!(resumed.state.status, runner.state.status);
6920        assert!(
6921            resumed.state.parked,
6922            "still parked until `execute` actually walks the graph again"
6923        );
6924        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
6925    }
6926
6927    /// A fresh open question on `run`, stored and handed back for assertions.
6928    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
6929        let mut q = ask::Question::new(
6930            run.to_owned(),
6931            "implement".to_owned(),
6932            "impl-A".to_owned(),
6933            "Which storage backend should the cache use?".to_owned(),
6934            String::new(),
6935            vec!["SQLite".to_owned(), "Redis".to_owned()],
6936        );
6937        store.put(&mut q).unwrap();
6938        q
6939    }
6940
6941    #[test]
6942    fn a_failed_runs_open_question_is_abandoned() {
6943        ask_test_home();
6944        let store = ask::Questions::open();
6945        let mut runner = runner_at(RunStatus::Failed);
6946        let run = runner.state.id.clone();
6947        let q = ask_open_question(&store, &run);
6948
6949        runner.settle_questions();
6950
6951        let back = store.get(&q.id).unwrap();
6952        assert!(
6953            !back.status.open(),
6954            "the seat that asked died with the run; nobody is left to read an answer"
6955        );
6956        assert!(
6957            back.detail.contains(&run) && back.detail.contains("failed"),
6958            "the reason names what the run became, not just that it is gone: {}",
6959            back.detail
6960        );
6961    }
6962
6963    #[test]
6964    fn a_merged_runs_open_question_is_abandoned_too() {
6965        ask_test_home();
6966        let store = ask::Questions::open();
6967        // A run that finishes cleanly still leaves nobody to read an answer -
6968        // this is not only a failure-path cleanup.
6969        for status in [RunStatus::Merged, RunStatus::Ready] {
6970            let mut runner = runner_at(status);
6971            let run = runner.state.id.clone();
6972            let q = ask_open_question(&store, &run);
6973
6974            runner.settle_questions();
6975
6976            let back = store.get(&q.id).unwrap();
6977            assert!(
6978                !back.status.open(),
6979                "{status:?} run's question must not outlive the run"
6980            );
6981        }
6982    }
6983
6984    #[test]
6985    fn a_still_resumable_runs_open_question_is_left_alone() {
6986        ask_test_home();
6987        let store = ask::Questions::open();
6988        // `Blocked` and `Stalled` can still be resumed — the candidates, the
6989        // review round and the seat sessions are all still on disk — so a
6990        // question asked mid-round may yet get a real answer from a real
6991        // resume. Sweeping it here would be exactly the failure mode this
6992        // whole feature exists to avoid on the other side.
6993        for status in [RunStatus::Blocked, RunStatus::Stalled] {
6994            let mut runner = runner_at(status);
6995            let run = runner.state.id.clone();
6996            let q = ask_open_question(&store, &run);
6997
6998            runner.settle_questions();
6999
7000            let back = store.get(&q.id).unwrap();
7001            assert!(
7002                back.status.open(),
7003                "{status:?} is still alive; the question must still be waiting"
7004            );
7005        }
7006    }
7007
7008    #[test]
7009    fn settle_questions_never_touches_an_already_answered_question() {
7010        ask_test_home();
7011        let store = ask::Questions::open();
7012        let mut runner = runner_at(RunStatus::Failed);
7013        let run = runner.state.id.clone();
7014        let mut q = ask_open_question(&store, &run);
7015        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
7016            .unwrap();
7017        store.put(&mut q).unwrap();
7018
7019        // Called twice, the way a crash-recovered daemon reclaim and the
7020        // graph's own cleanup both can for the same run — `abandon_for_run`
7021        // only ever touches what is still open, so this must be inert both
7022        // times, not merely the second.
7023        runner.settle_questions();
7024        runner.settle_questions();
7025
7026        let back = store.get(&q.id).unwrap();
7027        assert_eq!(
7028            back.status,
7029            ask::QuestionStatus::Answered,
7030            "a real answer is a decision on record, never overwritten by a sweep"
7031        );
7032    }
7033
7034    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
7035    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
7036    /// without merging, whose winner is still the operator's answer to read.
7037    /// Nothing previously called `fold_run` itself with a real `tally`, so
7038    /// this is the first test to pin down the one distinction the whole
7039    /// automatic-fold feature depends on: the winner's worktree and branch
7040    /// must survive, everything else sharing the run's worktree bay - a
7041    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
7042    /// second sweep treats every non-winner directory under the bay alike -
7043    /// must not.
7044    #[tokio::test]
7045    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
7046        crate::run::set_home(std::env::temp_dir().join("magi-graph-fold-run-tests-home"));
7047        let tmp = tempfile::tempdir().expect("tempdir");
7048        let repo = tmp.path().join("repo");
7049        std::fs::create_dir_all(&repo).unwrap();
7050        init_repo(&repo);
7051
7052        let mut config = Config::default();
7053        config.graph.worktree_root = Some(tmp.path().join("wt"));
7054
7055        let mut state = RunState::new(
7056            repo.clone(),
7057            "main".to_owned(),
7058            "deadbeef".to_owned(),
7059            "task".to_owned(),
7060            config,
7061        );
7062        let root = state.worktree_root();
7063        let wt_a = root.join("cand-A");
7064        let wt_b = root.join("cand-B");
7065        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
7066            .await
7067            .expect("worktree A");
7068        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
7069            .await
7070            .expect("worktree B");
7071
7072        state.candidates = vec![
7073            Candidate {
7074                index: 0,
7075                label: 'A',
7076                agent: "alpha".to_owned(),
7077                branch: "magi/x/A".to_owned(),
7078                worktree: wt_a.clone(),
7079                summary: String::new(),
7080                stat: String::new(),
7081                files: 0,
7082                commits: 0,
7083                empty: false,
7084                failed: None,
7085                verified_noop: None,
7086                duration_ms: 0,
7087                folded: false,
7088            },
7089            Candidate {
7090                index: 1,
7091                label: 'B',
7092                agent: "beta".to_owned(),
7093                branch: "magi/x/B".to_owned(),
7094                worktree: wt_b.clone(),
7095                summary: String::new(),
7096                stat: String::new(),
7097                files: 0,
7098                commits: 0,
7099                empty: false,
7100                failed: None,
7101                verified_noop: None,
7102                duration_ms: 0,
7103                folded: false,
7104            },
7105        ];
7106        state.tally = Some(Tally {
7107            first_choice: BTreeMap::from([('A', 1)]),
7108            borda: BTreeMap::new(),
7109            winner: 'A',
7110            rankings: 1,
7111            unanimous_initial: true,
7112            deliberated: false,
7113            changed_votes: 0,
7114            unanimous_final: true,
7115            tie_break: None,
7116            judges: 1,
7117            present: 1,
7118            quorum: 1,
7119            met_quorum: true,
7120            uncontested: None,
7121        });
7122        state.status = RunStatus::Ready;
7123
7124        fold_run(&mut state, false, &crate::run::home())
7125            .await
7126            .expect("fold_run");
7127
7128        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
7129        assert!(
7130            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
7131            "the unmerged winner's branch survives"
7132        );
7133        assert!(
7134            !state.candidates[0].folded,
7135            "the winner is not marked folded"
7136        );
7137
7138        assert!(!wt_b.exists(), "the loser's worktree is removed");
7139        assert!(
7140            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
7141            "the loser's branch is removed"
7142        );
7143        assert!(state.candidates[1].folded, "the loser is marked folded");
7144    }
7145
7146    /// `status == Ready` used to be read as "this is the harmless
7147    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
7148    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
7149    /// run's PR was closed without merging — and reentering `merge` with
7150    /// `mode` still `Pr` does not know the difference, so it pushed and
7151    /// opened a second pull request. `mode == Local` reproduces the same
7152    /// blind spot without a network call: reentry must not attempt another
7153    /// git merge once this node has already recorded an outcome.
7154    #[tokio::test]
7155    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
7156        let tmp = tempfile::tempdir().expect("tempdir");
7157        let repo = tmp.path().join("repo");
7158        std::fs::create_dir_all(&repo).unwrap();
7159        init_repo(&repo);
7160
7161        let mut config = Config::default();
7162        config.merge.mode = MergeMode::Local;
7163
7164        let mut state = RunState::new(
7165            repo.clone(),
7166            "main".to_owned(),
7167            "deadbeef".to_owned(),
7168            "task".to_owned(),
7169            config,
7170        );
7171        state.candidates = vec![Candidate {
7172            index: 0,
7173            label: 'A',
7174            agent: "alpha".to_owned(),
7175            branch: "does-not-exist".to_owned(),
7176            worktree: repo.clone(),
7177            summary: String::new(),
7178            stat: String::new(),
7179            files: 0,
7180            commits: 0,
7181            empty: false,
7182            failed: None,
7183            verified_noop: None,
7184            duration_ms: 0,
7185            folded: false,
7186        }];
7187        state.tally = Some(Tally {
7188            first_choice: BTreeMap::from([('A', 1)]),
7189            borda: BTreeMap::new(),
7190            winner: 'A',
7191            rankings: 1,
7192            unanimous_initial: true,
7193            deliberated: false,
7194            changed_votes: 0,
7195            unanimous_final: true,
7196            tie_break: None,
7197            judges: 0,
7198            present: 0,
7199            quorum: 0,
7200            met_quorum: true,
7201            uncontested: Some("only candidate A produced a change".to_owned()),
7202        });
7203        state.reviews = vec![ReviewRound {
7204            round: 1,
7205            head: "deadbeef".to_owned(),
7206            verified_head: None,
7207            verified_at: None,
7208            reviews: Vec::new(),
7209            e2e: Vec::new(),
7210            fix: None,
7211            blocking: 0,
7212            answered: 0,
7213            expected: 0,
7214            clean: true,
7215            verify_retried: false,
7216            e2e_deferred: false,
7217            e2e_defer_reason: None,
7218            progressed: false,
7219            vote_split: false,
7220            reconsideration: Vec::new(),
7221            verdict: None,
7222        }];
7223        state.gate = vec![CommandOutcome {
7224            command: "test".to_owned(),
7225            code: Some(0),
7226            output_tail: String::new(),
7227            duration_ms: 0,
7228            resource_blocked: false,
7229        }];
7230        state.gate_ran = true;
7231        // Reached its conclusion already — e.g. `land` closing the PR without
7232        // merging it, which (like the honest `MergeMode::None` path) leaves
7233        // `status` at `Ready`. The recorded outcome is what actually marks
7234        // this node done.
7235        state.status = RunStatus::Ready;
7236        state.merge = Some(MergeOutcome {
7237            mode: MergeMode::Local,
7238            ok: false,
7239            detail: "already concluded".to_owned(),
7240        });
7241
7242        let mut runner = Runner {
7243            state,
7244            roles: ResolvedRoles {
7245                implementers: Vec::new(),
7246                judges: Vec::new(),
7247                reviewers: Vec::new(),
7248                fixer: None,
7249                conductor: conductor(),
7250                implementer_roster: Vec::new(),
7251            },
7252            sem: Arc::new(Semaphore::new(1)),
7253            pause: Pause::new(),
7254            interrupt: Pause::new(),
7255        };
7256
7257        runner.merge().await.expect("merge");
7258
7259        assert_eq!(
7260            runner.state.status,
7261            RunStatus::Ready,
7262            "a concluded run's status must not change on reentry"
7263        );
7264        assert_eq!(
7265            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
7266            Some("already concluded"),
7267            "merge must not run again once the node already recorded an outcome"
7268        );
7269    }
7270
7271    /// `gate` leaves `state.gate_ran` false both before it has ever run and
7272    /// when its last attempt was resource-blocked (the shared build cache
7273    /// could not be acquired or confirmed fresh in time - see
7274    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
7275    /// `Vec` this also leaves behind used to read as "nothing failed" and let
7276    /// a run merge a tree the gate never actually checked - exactly the case
7277    /// a contended cache produces on every retry until it clears. `merge`
7278    /// must refuse until `gate` has actually recorded an attempt.
7279    #[tokio::test]
7280    async fn merge_refuses_a_gate_that_has_not_actually_run() {
7281        let tmp = tempfile::tempdir().expect("tempdir");
7282        let repo = tmp.path().join("repo");
7283        std::fs::create_dir_all(&repo).unwrap();
7284        init_repo(&repo);
7285
7286        let mut config = Config::default();
7287        config.merge.mode = MergeMode::Local;
7288
7289        let mut state = RunState::new(
7290            repo.clone(),
7291            "main".to_owned(),
7292            "deadbeef".to_owned(),
7293            "task".to_owned(),
7294            config,
7295        );
7296        state.candidates = vec![Candidate {
7297            index: 0,
7298            label: 'A',
7299            agent: "alpha".to_owned(),
7300            branch: "does-not-exist".to_owned(),
7301            worktree: repo.clone(),
7302            summary: String::new(),
7303            stat: String::new(),
7304            files: 0,
7305            commits: 0,
7306            empty: false,
7307            failed: None,
7308            verified_noop: None,
7309            duration_ms: 0,
7310            folded: false,
7311        }];
7312        state.tally = Some(Tally {
7313            first_choice: BTreeMap::from([('A', 1)]),
7314            borda: BTreeMap::new(),
7315            winner: 'A',
7316            rankings: 1,
7317            unanimous_initial: true,
7318            deliberated: false,
7319            changed_votes: 0,
7320            unanimous_final: true,
7321            tie_break: None,
7322            judges: 0,
7323            present: 0,
7324            quorum: 0,
7325            met_quorum: true,
7326            uncontested: Some("only candidate A produced a change".to_owned()),
7327        });
7328        state.reviews = vec![ReviewRound {
7329            round: 1,
7330            head: "deadbeef".to_owned(),
7331            verified_head: None,
7332            verified_at: None,
7333            reviews: Vec::new(),
7334            e2e: Vec::new(),
7335            fix: None,
7336            blocking: 0,
7337            answered: 0,
7338            expected: 0,
7339            clean: true,
7340            verify_retried: false,
7341            e2e_deferred: false,
7342            e2e_defer_reason: None,
7343            progressed: false,
7344            vote_split: false,
7345            reconsideration: Vec::new(),
7346            verdict: None,
7347        }];
7348        // The point: `gate` has not recorded anything yet.
7349        state.gate = Vec::new();
7350        state.gate_ran = false;
7351        state.status = RunStatus::Gating;
7352
7353        let mut runner = Runner {
7354            state,
7355            roles: ResolvedRoles {
7356                implementers: Vec::new(),
7357                judges: Vec::new(),
7358                reviewers: Vec::new(),
7359                fixer: None,
7360                conductor: conductor(),
7361                implementer_roster: Vec::new(),
7362            },
7363            sem: Arc::new(Semaphore::new(1)),
7364            pause: Pause::new(),
7365            interrupt: Pause::new(),
7366        };
7367
7368        runner.merge().await.expect("merge");
7369
7370        assert!(
7371            runner.state.merge.is_none(),
7372            "an empty gate must never be read as a passing one: {:?}",
7373            runner.state.merge
7374        );
7375    }
7376
7377    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
7378    /// commands configured and `merge.mode` is `none` (a review-only run).
7379    /// `gate` must still record a real attempt — zero commands, vacuously
7380    /// passed — rather than leaving `state.gate` empty in a way `merge`
7381    /// cannot tell apart from "never ran"; otherwise the run reaches
7382    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
7383    #[tokio::test]
7384    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
7385        let tmp = tempfile::tempdir().expect("tempdir");
7386        let repo = tmp.path().join("repo");
7387        std::fs::create_dir_all(&repo).unwrap();
7388        init_repo(&repo);
7389
7390        // Default config: `verify.gate` empty, `merge.mode` is `none`.
7391        let config = Config::default();
7392
7393        let mut state = RunState::new(
7394            repo.clone(),
7395            "main".to_owned(),
7396            "deadbeef".to_owned(),
7397            "task".to_owned(),
7398            config,
7399        );
7400        state.candidates = vec![Candidate {
7401            index: 0,
7402            label: 'A',
7403            agent: "alpha".to_owned(),
7404            branch: "does-not-exist".to_owned(),
7405            worktree: repo.clone(),
7406            summary: String::new(),
7407            stat: String::new(),
7408            files: 0,
7409            commits: 0,
7410            empty: false,
7411            failed: None,
7412            verified_noop: None,
7413            duration_ms: 0,
7414            folded: false,
7415        }];
7416        state.tally = Some(Tally {
7417            first_choice: BTreeMap::from([('A', 1)]),
7418            borda: BTreeMap::new(),
7419            winner: 'A',
7420            rankings: 1,
7421            unanimous_initial: true,
7422            deliberated: false,
7423            changed_votes: 0,
7424            unanimous_final: true,
7425            tie_break: None,
7426            judges: 0,
7427            present: 0,
7428            quorum: 0,
7429            met_quorum: true,
7430            uncontested: Some("only candidate A produced a change".to_owned()),
7431        });
7432        state.reviews = vec![ReviewRound {
7433            round: 1,
7434            head: "deadbeef".to_owned(),
7435            verified_head: None,
7436            verified_at: None,
7437            reviews: Vec::new(),
7438            e2e: Vec::new(),
7439            fix: None,
7440            blocking: 0,
7441            answered: 0,
7442            expected: 0,
7443            clean: true,
7444            verify_retried: false,
7445            e2e_deferred: false,
7446            e2e_defer_reason: None,
7447            progressed: false,
7448            vote_split: false,
7449            reconsideration: Vec::new(),
7450            verdict: None,
7451        }];
7452
7453        let mut runner = Runner {
7454            state,
7455            roles: ResolvedRoles {
7456                implementers: Vec::new(),
7457                judges: Vec::new(),
7458                reviewers: Vec::new(),
7459                fixer: None,
7460                conductor: conductor(),
7461                implementer_roster: Vec::new(),
7462            },
7463            sem: Arc::new(Semaphore::new(1)),
7464            pause: Pause::new(),
7465            interrupt: Pause::new(),
7466        };
7467
7468        runner.gate().await.expect("gate");
7469        assert!(
7470            runner.state.gate_ran,
7471            "zero configured commands is still a real attempt, not an unrun gate"
7472        );
7473        assert!(runner.state.gate.is_empty());
7474        assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
7475        assert_ne!(
7476            runner.state.status,
7477            RunStatus::Blocked,
7478            "a gate with nothing to check must not read as failed"
7479        );
7480
7481        runner.merge().await.expect("merge");
7482        assert_eq!(
7483            runner.state.status,
7484            RunStatus::Ready,
7485            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
7486        );
7487    }
7488
7489    /// `Config::cache_dir` is derived from `verify.e2e` as well as
7490    /// `verify.gate` (so the e2e leg and the final gate never build against
7491    /// different directories). With zero `verify.gate` commands but a
7492    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
7493    /// that lease before discovering it had nothing to run - so a repo with
7494    /// no gate commands could come back `resource_blocked` (and therefore
7495    /// still `gate_ran == false`) on nothing but an unrelated run holding the
7496    /// cache, exactly the contention this run's own zero commands could
7497    /// never have touched. `gate` must recognise there is nothing to check
7498    /// before it ever asks for the lease.
7499    #[tokio::test]
7500    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
7501        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
7502        let home = crate::run::home();
7503
7504        let tmp = tempfile::tempdir().expect("tempdir");
7505        let repo = tmp.path().join("repo");
7506        std::fs::create_dir_all(&repo).unwrap();
7507        init_repo(&repo);
7508        // Unique to this test, so holding its lease cannot collide with
7509        // another test sharing the same process-wide `home`.
7510        let cache_dir = tmp.path().join("target");
7511
7512        let mut config = Config::default();
7513        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
7514        // `verify.gate` stays empty (the default). Bounded so a regression
7515        // that does start waiting fails the test in seconds, not hangs it.
7516        config.graph.timeout_verify = Some(2);
7517
7518        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
7519        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
7520            .expect("no io error acquiring directly")
7521        {
7522            crate::cache::AcquireOutcome::Acquired(g) => g,
7523            crate::cache::AcquireOutcome::Busy(b) => {
7524                panic!("expected the direct acquire to win the lease first: {b:?}")
7525            }
7526        };
7527
7528        let mut state = RunState::new(
7529            repo.clone(),
7530            "main".to_owned(),
7531            "deadbeef".to_owned(),
7532            "task".to_owned(),
7533            config,
7534        );
7535        state.candidates = vec![Candidate {
7536            index: 0,
7537            label: 'A',
7538            agent: "alpha".to_owned(),
7539            branch: "does-not-exist".to_owned(),
7540            worktree: repo.clone(),
7541            summary: String::new(),
7542            stat: String::new(),
7543            files: 0,
7544            commits: 0,
7545            empty: false,
7546            failed: None,
7547            verified_noop: None,
7548            duration_ms: 0,
7549            folded: false,
7550        }];
7551        state.tally = Some(Tally {
7552            first_choice: BTreeMap::from([('A', 1)]),
7553            borda: BTreeMap::new(),
7554            winner: 'A',
7555            rankings: 1,
7556            unanimous_initial: true,
7557            deliberated: false,
7558            changed_votes: 0,
7559            unanimous_final: true,
7560            tie_break: None,
7561            judges: 0,
7562            present: 0,
7563            quorum: 0,
7564            met_quorum: true,
7565            uncontested: Some("only candidate A produced a change".to_owned()),
7566        });
7567        state.reviews = vec![ReviewRound {
7568            round: 1,
7569            head: "deadbeef".to_owned(),
7570            verified_head: None,
7571            verified_at: None,
7572            reviews: Vec::new(),
7573            e2e: Vec::new(),
7574            fix: None,
7575            blocking: 0,
7576            answered: 0,
7577            expected: 0,
7578            clean: true,
7579            verify_retried: false,
7580            e2e_deferred: false,
7581            e2e_defer_reason: None,
7582            progressed: false,
7583            vote_split: false,
7584            reconsideration: Vec::new(),
7585            verdict: None,
7586        }];
7587
7588        let mut runner = Runner {
7589            state,
7590            roles: ResolvedRoles {
7591                implementers: Vec::new(),
7592                judges: Vec::new(),
7593                reviewers: Vec::new(),
7594                fixer: None,
7595                conductor: conductor(),
7596                implementer_roster: Vec::new(),
7597            },
7598            sem: Arc::new(Semaphore::new(1)),
7599            pause: Pause::new(),
7600            interrupt: Pause::new(),
7601        };
7602
7603        let started = std::time::Instant::now();
7604        runner.gate().await.expect("gate");
7605        assert!(
7606            started.elapsed() < Duration::from_secs(1),
7607            "a gate with nothing to run must never wait on a lease it never needed"
7608        );
7609        assert!(
7610            runner.state.gate_ran,
7611            "zero commands is still a real, immediate attempt"
7612        );
7613        assert!(runner.state.gate.is_empty());
7614        assert_ne!(
7615            runner.state.status,
7616            RunStatus::Blocked,
7617            "must not read as resource-blocked on a lease it never asked for"
7618        );
7619    }
7620
7621    /// The addendum's second gap: a `verify.gate` command running for real
7622    /// wall-clock time had nothing at all to show for it in `active` before
7623    /// `run_commands` learned to record it — a run could sit in `Gating` for
7624    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
7625    /// what was actually happening. Proven with a genuinely still-running
7626    /// command, not just a before/after check on the final state: a poller
7627    /// task reads the same `run.json` `gate()` is writing, the same way the
7628    /// phone or `magi show` would, while the shell command is still blocked
7629    /// on its own release marker.
7630    #[tokio::test]
7631    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
7632        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
7633
7634        let tmp = tempfile::tempdir().expect("tempdir");
7635        let repo = tmp.path().join("repo");
7636        std::fs::create_dir_all(&repo).unwrap();
7637        init_repo(&repo);
7638
7639        let mut config = Config::default();
7640        config.verify.gate = vec![
7641            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
7642             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
7643                .to_owned(),
7644        ];
7645
7646        let mut state = RunState::new(
7647            repo.clone(),
7648            "main".to_owned(),
7649            "deadbeef".to_owned(),
7650            "task".to_owned(),
7651            config,
7652        );
7653        let run_id = state.id.clone();
7654        state.candidates = vec![Candidate {
7655            index: 0,
7656            label: 'A',
7657            agent: "alpha".to_owned(),
7658            branch: "does-not-exist".to_owned(),
7659            worktree: repo.clone(),
7660            summary: String::new(),
7661            stat: String::new(),
7662            files: 0,
7663            commits: 0,
7664            empty: false,
7665            failed: None,
7666            verified_noop: None,
7667            duration_ms: 0,
7668            folded: false,
7669        }];
7670        state.tally = Some(Tally {
7671            first_choice: BTreeMap::from([('A', 1)]),
7672            borda: BTreeMap::new(),
7673            winner: 'A',
7674            rankings: 1,
7675            unanimous_initial: true,
7676            deliberated: false,
7677            changed_votes: 0,
7678            unanimous_final: true,
7679            tie_break: None,
7680            judges: 0,
7681            present: 0,
7682            quorum: 0,
7683            met_quorum: true,
7684            uncontested: Some("only candidate A produced a change".to_owned()),
7685        });
7686        state.reviews = vec![ReviewRound {
7687            round: 1,
7688            head: "deadbeef".to_owned(),
7689            verified_head: None,
7690            verified_at: None,
7691            reviews: Vec::new(),
7692            e2e: Vec::new(),
7693            fix: None,
7694            blocking: 0,
7695            answered: 0,
7696            expected: 0,
7697            clean: true,
7698            verify_retried: false,
7699            e2e_deferred: false,
7700            e2e_defer_reason: None,
7701            progressed: false,
7702            vote_split: false,
7703            reconsideration: Vec::new(),
7704            verdict: None,
7705        }];
7706
7707        let mut runner = Runner {
7708            state,
7709            roles: ResolvedRoles {
7710                implementers: Vec::new(),
7711                judges: Vec::new(),
7712                reviewers: Vec::new(),
7713                fixer: None,
7714                conductor: conductor(),
7715                implementer_roster: Vec::new(),
7716            },
7717            sem: Arc::new(Semaphore::new(1)),
7718            pause: Pause::new(),
7719            interrupt: Pause::new(),
7720        };
7721
7722        let started_marker = repo.join("started.marker");
7723        let release_marker = repo.join("release.marker");
7724        let poller = tokio::spawn(async move {
7725            // Bounded so a regression that never records the task entry
7726            // fails this test in seconds instead of hanging the suite —
7727            // the same shape `a_park_requested_while_a_seat_is_mid_call_
7728            // does_not_cut_it_short` uses for the same reason.
7729            for _ in 0..100 {
7730                if started_marker.exists()
7731                    && let Ok(s) = crate::run::RunState::load(&run_id)
7732                    && let Some(a) = s.active.get("gate")
7733                {
7734                    std::fs::write(&release_marker, b"go").expect("release marker");
7735                    return Some(a.clone());
7736                }
7737                tokio::time::sleep(Duration::from_millis(50)).await;
7738            }
7739            None
7740        });
7741
7742        runner.gate().await.expect("gate");
7743        let captured = poller.await.expect("poller task");
7744        let captured = captured.expect(
7745            "the poller never saw a `gate` task entry in run.json while the command was \
7746             still blocked on its own release marker",
7747        );
7748
7749        assert_eq!(captured.task.as_deref(), Some("gate"));
7750        assert_eq!(captured.node, "gate");
7751        assert_eq!(captured.index, Some(1));
7752        assert_eq!(captured.total, Some(1));
7753        assert!(
7754            captured
7755                .command
7756                .as_deref()
7757                .is_some_and(|c| c.contains("started.marker")),
7758            "{captured:?}"
7759        );
7760
7761        assert!(
7762            runner.state.active.is_empty(),
7763            "the entry must be cleared once the command actually finished: {:?}",
7764            runner.state.active
7765        );
7766        assert!(runner.state.gate_ran);
7767        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
7768    }
7769
7770    /// The shape the incident this whole fix responds to actually had: the
7771    /// round budget spent, the last round's own e2e blocked on the shared
7772    /// build cache (held here by a live pid — this test process — exactly
7773    /// `cache`'s own unit tests' pattern for "another owner, still alive"
7774    /// without forking a process). `stop_reviewing` must retry it — not
7775    /// silently leave the round looking untouched (the catch-up-only half of
7776    /// the bug), and not read the contention as a red `e2e` and block the
7777    /// run on it (the other half). Called directly, the same way
7778    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
7779    /// above exercises `gate`, so this never needs a real cargo build to
7780    /// reach: the lease is never released, so `with_cache_lease` never gets
7781    /// past acquiring it into anything that would need a real workspace.
7782    #[tokio::test]
7783    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
7784        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
7785        let home = crate::run::home();
7786
7787        let tmp = tempfile::tempdir().expect("tempdir");
7788        let repo = tmp.path().join("repo");
7789        std::fs::create_dir_all(&repo).unwrap();
7790        init_repo(&repo);
7791        let head = crate::git::rev_parse(&repo, "HEAD")
7792            .await
7793            .expect("rev-parse");
7794        // Unique to this test, so holding its lease cannot collide with
7795        // another test sharing the same process-wide `home`.
7796        let cache_dir = tmp.path().join("target");
7797
7798        let mut config = Config::default();
7799        config.verify.e2e = vec![format!(
7800            "CARGO_TARGET_DIR='{}' test -f README.md",
7801            cache_dir.display()
7802        )];
7803        config.graph.review_rounds = 1;
7804        // Bounded so a regression that does start waiting fails the test in
7805        // seconds, not hangs it.
7806        config.graph.timeout_verify = Some(2);
7807
7808        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
7809        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
7810            .expect("no io error acquiring directly")
7811        {
7812            crate::cache::AcquireOutcome::Acquired(g) => g,
7813            crate::cache::AcquireOutcome::Busy(b) => {
7814                panic!("expected the direct acquire to win the lease first: {b:?}")
7815            }
7816        };
7817
7818        let mut state = RunState::new(
7819            repo.clone(),
7820            "main".to_owned(),
7821            head.clone(),
7822            "task".to_owned(),
7823            config,
7824        );
7825        state.candidates = vec![Candidate {
7826            index: 0,
7827            label: 'A',
7828            agent: "alpha".to_owned(),
7829            branch: "does-not-exist".to_owned(),
7830            worktree: repo.clone(),
7831            summary: String::new(),
7832            stat: String::new(),
7833            files: 0,
7834            commits: 0,
7835            empty: false,
7836            failed: None,
7837            verified_noop: None,
7838            duration_ms: 0,
7839            folded: false,
7840        }];
7841        state.tally = Some(Tally {
7842            first_choice: BTreeMap::from([('A', 1)]),
7843            borda: BTreeMap::new(),
7844            winner: 'A',
7845            rankings: 1,
7846            unanimous_initial: true,
7847            deliberated: false,
7848            changed_votes: 0,
7849            unanimous_final: true,
7850            tie_break: None,
7851            judges: 0,
7852            present: 0,
7853            quorum: 0,
7854            met_quorum: true,
7855            uncontested: Some("only candidate A produced a change".to_owned()),
7856        });
7857        // The round budget's last round, deferred: `needs_catchup_run`'s
7858        // other trigger. `stop_reviewing`'s retry machinery must treat this
7859        // exactly like a resource-blocked attempt once it actually runs.
7860        state.reviews = vec![ReviewRound {
7861            round: 1,
7862            head: head.clone(),
7863            verified_head: None,
7864            verified_at: None,
7865            reviews: Vec::new(),
7866            e2e: Vec::new(),
7867            fix: None,
7868            blocking: 1,
7869            answered: 1,
7870            expected: 1,
7871            clean: false,
7872            verify_retried: false,
7873            e2e_deferred: true,
7874            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
7875            progressed: false,
7876            vote_split: false,
7877            reconsideration: Vec::new(),
7878            verdict: None,
7879        }];
7880
7881        let mut runner = Runner {
7882            state,
7883            roles: ResolvedRoles {
7884                implementers: Vec::new(),
7885                judges: Vec::new(),
7886                reviewers: Vec::new(),
7887                fixer: None,
7888                conductor: conductor(),
7889                implementer_roster: Vec::new(),
7890            },
7891            sem: Arc::new(Semaphore::new(1)),
7892            pause: Pause::new(),
7893            interrupt: Pause::new(),
7894        };
7895
7896        let shell = runner.state.config.shell();
7897        runner
7898            .stop_reviewing("round budget spent", &shell, &repo)
7899            .await
7900            .expect("stop_reviewing");
7901
7902        let last = runner.state.reviews.last().expect("round record");
7903        assert_eq!(
7904            last.e2e_status(),
7905            E2eStatus::ResourceBlocked,
7906            "the shared cache is still held; the attempt must read as blocked, not deferred or \
7907             failed: {last:?}"
7908        );
7909        assert_eq!(
7910            last.verified_head.as_deref(),
7911            Some(head.as_str()),
7912            "which commit this attempt targeted is known even though nothing finished checking \
7913             it"
7914        );
7915        let first_attempt_at = last
7916            .verified_at
7917            .expect("when this attempt ran is known too");
7918        assert_ne!(
7919            runner.state.status,
7920            RunStatus::Blocked,
7921            "contention is evidence about the machine, not the patch — it must not settle the \
7922             run as blocked: {:?}",
7923            runner.state.status
7924        );
7925        assert!(
7926            !runner
7927                .state
7928                .events
7929                .iter()
7930                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
7931            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
7932            runner.state.events
7933        );
7934
7935        // The cache is still held: a later reentry must retry the same
7936        // round's verification again — not leave it looking exactly as
7937        // untouched as the first blocked attempt, which is indistinguishable
7938        // from never having tried again at all.
7939        runner
7940            .stop_reviewing("round budget spent", &shell, &repo)
7941            .await
7942            .expect("stop_reviewing retry");
7943        assert_eq!(
7944            runner.state.reviews.len(),
7945            1,
7946            "no new round was started: {:?}",
7947            runner.state.reviews
7948        );
7949        let last = runner.state.reviews.last().expect("round record");
7950        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
7951        assert!(
7952            last.verified_at.expect("still known") > first_attempt_at,
7953            "a second reentry must be a fresh attempt, not a stale copy of the first"
7954        );
7955        assert_ne!(runner.state.status, RunStatus::Blocked);
7956
7957        held.release();
7958    }
7959
7960    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
7961    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
7962    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
7963    /// own top-of-function fast path (`review_conclusion`) correctly reads
7964    /// this shape as `None` rather than guessing `Blocked`, and the loop's
7965    /// own `for` range is empty once the round budget is spent, so
7966    /// `review_loop` must retry the check itself rather than silently doing
7967    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
7968    /// above exercises directly, but through `review_loop`'s own entry point
7969    /// this time, proving the wiring between the two rather than just the
7970    /// retry logic in isolation.
7971    #[tokio::test]
7972    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
7973        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
7974        let home = crate::run::home();
7975
7976        let tmp = tempfile::tempdir().expect("tempdir");
7977        let repo = tmp.path().join("repo");
7978        std::fs::create_dir_all(&repo).unwrap();
7979        init_repo(&repo);
7980        let head = crate::git::rev_parse(&repo, "HEAD")
7981            .await
7982            .expect("rev-parse");
7983        let cache_dir = tmp.path().join("target");
7984
7985        let mut config = Config::default();
7986        config.verify.e2e = vec![format!(
7987            "CARGO_TARGET_DIR='{}' test -f README.md",
7988            cache_dir.display()
7989        )];
7990        config.graph.review_rounds = 1;
7991        config.graph.timeout_verify = Some(2);
7992
7993        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
7994        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
7995            .expect("no io error acquiring directly")
7996        {
7997            crate::cache::AcquireOutcome::Acquired(g) => g,
7998            crate::cache::AcquireOutcome::Busy(b) => {
7999                panic!("expected the direct acquire to win the lease first: {b:?}")
8000            }
8001        };
8002
8003        let mut state = RunState::new(
8004            repo.clone(),
8005            "main".to_owned(),
8006            head.clone(),
8007            "task".to_owned(),
8008            config,
8009        );
8010        state.candidates = vec![Candidate {
8011            index: 0,
8012            label: 'A',
8013            agent: "alpha".to_owned(),
8014            branch: "does-not-exist".to_owned(),
8015            worktree: repo.clone(),
8016            summary: String::new(),
8017            stat: String::new(),
8018            files: 0,
8019            commits: 0,
8020            empty: false,
8021            failed: None,
8022            verified_noop: None,
8023            duration_ms: 0,
8024            folded: false,
8025        }];
8026        state.tally = Some(Tally {
8027            first_choice: BTreeMap::from([('A', 1)]),
8028            borda: BTreeMap::new(),
8029            winner: 'A',
8030            rankings: 1,
8031            unanimous_initial: true,
8032            deliberated: false,
8033            changed_votes: 0,
8034            unanimous_final: true,
8035            tie_break: None,
8036            judges: 0,
8037            present: 0,
8038            quorum: 0,
8039            met_quorum: true,
8040            uncontested: Some("only candidate A produced a change".to_owned()),
8041        });
8042        // The exact shape a prior process's `stop_reviewing` would have left
8043        // on disk: the round budget's last round, a real attempt already
8044        // made and already resource-blocked.
8045        state.reviews = vec![ReviewRound {
8046            round: 1,
8047            head: head.clone(),
8048            verified_head: Some(head.clone()),
8049            verified_at: Some(jiff::Timestamp::now()),
8050            reviews: Vec::new(),
8051            e2e: vec![CommandOutcome {
8052                command: format!(
8053                    "CARGO_TARGET_DIR='{}' test -f README.md",
8054                    cache_dir.display()
8055                ),
8056                code: None,
8057                output_tail: "waiting for the shared build cache".to_owned(),
8058                duration_ms: 0,
8059                resource_blocked: true,
8060            }],
8061            fix: None,
8062            blocking: 1,
8063            answered: 1,
8064            expected: 1,
8065            clean: false,
8066            verify_retried: false,
8067            e2e_deferred: false,
8068            e2e_defer_reason: None,
8069            progressed: false,
8070            vote_split: false,
8071            reconsideration: Vec::new(),
8072            verdict: None,
8073        }];
8074
8075        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
8076        let mut runner = Runner {
8077            state,
8078            roles: ResolvedRoles {
8079                implementers: Vec::new(),
8080                judges: Vec::new(),
8081                reviewers: Vec::new(),
8082                fixer: None,
8083                conductor: conductor(),
8084                implementer_roster: Vec::new(),
8085            },
8086            sem: Arc::new(Semaphore::new(1)),
8087            pause: Pause::new(),
8088            interrupt: Pause::new(),
8089        };
8090
8091        // The lease is still held throughout, so this reentry's own retry is
8092        // also contended — proving `review_loop` actually tried again (not
8093        // that it happened to succeed) is what the timestamp comparison
8094        // below is for.
8095        runner.review_loop().await.expect("review_loop");
8096
8097        assert_eq!(
8098            runner.state.reviews.len(),
8099            1,
8100            "no new round was started on top of the unresolved one: {:?}",
8101            runner.state.reviews
8102        );
8103        let last = &runner.state.reviews[0];
8104        assert_eq!(
8105            last.e2e_status(),
8106            E2eStatus::ResourceBlocked,
8107            "still contended: {last:?}"
8108        );
8109        assert!(
8110            last.verified_at.expect("still known") > first_attempt_at,
8111            "review_loop must have actually retried the check, not left it exactly as found"
8112        );
8113        assert_ne!(
8114            runner.state.status,
8115            RunStatus::Blocked,
8116            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
8117            runner.state.status
8118        );
8119
8120        held.release();
8121    }
8122
8123    #[tokio::test]
8124    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
8125        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8126        let tmp = tempfile::tempdir().expect("tempdir");
8127        let repo = tmp.path().join("repo");
8128        std::fs::create_dir_all(&repo).unwrap();
8129        init_repo(&repo);
8130
8131        let mut config = Config::default();
8132        config.merge.mode = MergeMode::Pr;
8133        config.graph.land = true;
8134        config.graph.land_approval = false;
8135
8136        let mut state = RunState::new(
8137            repo.clone(),
8138            "main".to_owned(),
8139            "deadbeef".to_owned(),
8140            "task".to_owned(),
8141            config,
8142        );
8143        state.candidates = vec![Candidate {
8144            index: 0,
8145            label: 'A',
8146            agent: "alpha".to_owned(),
8147            branch: "does-not-exist".to_owned(),
8148            worktree: repo.clone(),
8149            summary: String::new(),
8150            stat: String::new(),
8151            files: 0,
8152            commits: 0,
8153            empty: false,
8154            failed: None,
8155            verified_noop: None,
8156            duration_ms: 0,
8157            folded: false,
8158        }];
8159        state.tally = Some(Tally {
8160            first_choice: BTreeMap::from([('A', 1)]),
8161            borda: BTreeMap::new(),
8162            winner: 'A',
8163            rankings: 1,
8164            unanimous_initial: true,
8165            deliberated: false,
8166            changed_votes: 0,
8167            unanimous_final: true,
8168            tie_break: None,
8169            judges: 0,
8170            present: 0,
8171            quorum: 0,
8172            met_quorum: true,
8173            uncontested: Some("only candidate A produced a change".to_owned()),
8174        });
8175        state.reviews = vec![ReviewRound {
8176            round: 1,
8177            head: "deadbeef".to_owned(),
8178            verified_head: None,
8179            verified_at: None,
8180            reviews: Vec::new(),
8181            e2e: Vec::new(),
8182            fix: None,
8183            blocking: 0,
8184            answered: 0,
8185            expected: 0,
8186            clean: true,
8187            verify_retried: false,
8188            e2e_deferred: false,
8189            e2e_defer_reason: None,
8190            progressed: false,
8191            vote_split: false,
8192            reconsideration: Vec::new(),
8193            verdict: None,
8194        }];
8195        state.gate = vec![CommandOutcome {
8196            command: "test".to_owned(),
8197            code: Some(0),
8198            output_tail: String::new(),
8199            duration_ms: 0,
8200            resource_blocked: false,
8201        }];
8202        state.gate_ran = true;
8203        // A first pass through `merge` already pushed and opened this pull
8204        // request; `status` is `Landing` because a previous call into `land`
8205        // parked or was interrupted before it reached a terminal outcome.
8206        state.status = RunStatus::Landing;
8207        state.merge = Some(MergeOutcome {
8208            mode: MergeMode::Pr,
8209            ok: true,
8210            detail: "https://example.invalid/x/y/pull/1".to_owned(),
8211        });
8212
8213        // The Landing-resume shortcut calls `run_land` directly rather than
8214        // through `merge`, which is exactly the call site that used to skip
8215        // `settle_questions` - see the fixture below.
8216        ask_test_home();
8217        let store = ask::Questions::open();
8218        let q = ask_open_question(&store, &state.id);
8219
8220        let mut runner = Runner {
8221            state,
8222            roles: ResolvedRoles {
8223                implementers: Vec::new(),
8224                judges: Vec::new(),
8225                reviewers: Vec::new(),
8226                fixer: None,
8227                conductor: conductor(),
8228                implementer_roster: Vec::new(),
8229            },
8230            sem: Arc::new(Semaphore::new(1)),
8231            pause: Pause::new(),
8232            interrupt: Pause::new(),
8233        };
8234
8235        // `execute`, not `merge` directly: the Landing-resume shortcut lives
8236        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
8237        // exactly because `review_loop` would otherwise clobber the marker
8238        // first.
8239        runner.execute().await.expect("execute");
8240
8241        assert_eq!(
8242            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
8243            Some("https://example.invalid/x/y/pull/1"),
8244            "reentry must not push again or open a second pull request over the \
8245             one `land` is already watching"
8246        );
8247        assert_ne!(
8248            runner.state.status,
8249            RunStatus::Landing,
8250            "land could not actually reach the fake pull request, so it must \
8251             have given up rather than left the run silently parked forever"
8252        );
8253        // `land` could not reach the fake pull request, so it gave up into
8254        // `Blocked` - still resumable, so the question must not have been
8255        // swept just because this branch now also calls `settle_questions`.
8256        assert_eq!(runner.state.status, RunStatus::Blocked);
8257        assert!(
8258            store.get(&q.id).unwrap().status.open(),
8259            "Blocked is still alive; settle_questions must have been a no-op here"
8260        );
8261    }
8262
8263    fn state_with_round(round: ReviewRound) -> RunState {
8264        let mut s = RunState::new(
8265            PathBuf::from("/repo"),
8266            "main".to_owned(),
8267            "abc1234".to_owned(),
8268            "add retries".to_owned(),
8269            Config::default(),
8270        );
8271        s.reviews = vec![round];
8272        s
8273    }
8274
8275    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
8276        crate::verdict::Finding {
8277            id: id.to_owned(),
8278            severity,
8279            file: None,
8280            line: None,
8281            title: title.to_owned(),
8282            detail: String::new(),
8283        }
8284    }
8285
8286    #[test]
8287    fn pr_body_names_open_findings_and_declined_ones() {
8288        let round = ReviewRound {
8289            round: 2,
8290            head: "deadbee".to_owned(),
8291            verified_head: None,
8292            verified_at: None,
8293            reviews: vec![ReviewRecord {
8294                attempts: 0,
8295                reviewer: 1,
8296                agent: "alpha".to_owned(),
8297                summary: String::new(),
8298                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
8299                vote: None,
8300                failed: None,
8301                duration_ms: 0,
8302            }],
8303            e2e: vec![CommandOutcome {
8304                command: "cargo test".to_owned(),
8305                code: Some(0),
8306                output_tail: String::new(),
8307                duration_ms: 0,
8308                resource_blocked: false,
8309            }],
8310            verify_retried: false,
8311            e2e_deferred: false,
8312            e2e_defer_reason: None,
8313            fix: Some(FixRecord {
8314                agent: "alpha".to_owned(),
8315                addressed: Vec::new(),
8316                rejected: vec![crate::verdict::Rejection {
8317                    id: "R1-1-1".to_owned(),
8318                    why: "not reachable from any caller".to_owned(),
8319                }],
8320                notes: String::new(),
8321                committed: true,
8322                failed: None,
8323                duration_ms: 0,
8324                continuation: None,
8325            }),
8326            blocking: 0,
8327            answered: 1,
8328            expected: 1,
8329            clean: false,
8330            progressed: true,
8331            vote_split: false,
8332            reconsideration: Vec::new(),
8333            verdict: None,
8334        };
8335        let state = state_with_round(round);
8336        let body = pr_body(&state, 'A');
8337
8338        assert!(body.contains("add retries"), "the task must still be there");
8339        assert!(body.contains("R2-1-1"), "{body}");
8340        assert!(body.contains("unused import"), "{body}");
8341        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
8342        assert!(
8343            body.contains("not reachable from any caller"),
8344            "the reason it was declined: {body}"
8345        );
8346    }
8347
8348    #[test]
8349    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
8350        let round = ReviewRound {
8351            round: 1,
8352            head: "deadbee".to_owned(),
8353            verified_head: None,
8354            verified_at: None,
8355            reviews: vec![ReviewRecord {
8356                attempts: 0,
8357                reviewer: 1,
8358                agent: "alpha".to_owned(),
8359                summary: String::new(),
8360                findings: Vec::new(),
8361                vote: None,
8362                failed: None,
8363                duration_ms: 0,
8364            }],
8365            e2e: Vec::new(),
8366            verify_retried: false,
8367            e2e_deferred: false,
8368            e2e_defer_reason: None,
8369            fix: None,
8370            blocking: 0,
8371            answered: 1,
8372            expected: 1,
8373            clean: true,
8374            progressed: false,
8375            vote_split: false,
8376            reconsideration: Vec::new(),
8377            verdict: None,
8378        };
8379        let state = state_with_round(round);
8380        let body = pr_body(&state, 'A');
8381        assert!(!body.contains("Open review findings"), "{body}");
8382        assert!(!body.contains("Declined"), "{body}");
8383    }
8384
8385    #[test]
8386    fn pr_body_titles_itself_from_the_task_not_run_or_candidate() {
8387        let state = RunState::new(
8388            PathBuf::from("/repo"),
8389            "main".to_owned(),
8390            "abc1234".to_owned(),
8391            "add retries".to_owned(),
8392            Config::default(),
8393        );
8394        let body = pr_body(&state, 'A');
8395        let title = body.lines().next().unwrap();
8396
8397        assert_eq!(
8398            title, "add retries",
8399            "the title must be the task, not run/candidate bookkeeping: {body}"
8400        );
8401        assert!(
8402            body.contains(&format!("magi:run/{}", state.id)),
8403            "the run id must still be recoverable from the footer: {body}"
8404        );
8405        assert!(
8406            body.contains("magi:candidate-a"),
8407            "the candidate must still be recoverable from the footer: {body}"
8408        );
8409    }
8410
8411    #[test]
8412    fn pr_body_never_titles_itself_off_a_blank_first_line() {
8413        let leading_blank = RunState::new(
8414            PathBuf::from("/repo"),
8415            "main".to_owned(),
8416            "abc1234".to_owned(),
8417            "\n\n  \nadd retries\n\ndetails".to_owned(),
8418            Config::default(),
8419        );
8420        let body = pr_body(&leading_blank, 'A');
8421        assert_eq!(
8422            body.lines().next(),
8423            Some("add retries"),
8424            "a leading blank line must not become an empty title: {body}"
8425        );
8426
8427        let whitespace_only = RunState::new(
8428            PathBuf::from("/repo"),
8429            "main".to_owned(),
8430            "abc1234".to_owned(),
8431            "   \n  \n".to_owned(),
8432            Config::default(),
8433        );
8434        let body = pr_body(&whitespace_only, 'A');
8435        let title = body.lines().next().unwrap_or_default();
8436        assert!(
8437            !title.is_empty(),
8438            "a whitespace-only instruction must still fall back to a non-empty title: {body}"
8439        );
8440    }
8441
8442    #[test]
8443    fn pr_title_truncates_a_first_line_over_githubs_limit() {
8444        // A run 2963-shaped instruction: a single first line well past
8445        // GitHub's 256-character createPullRequest limit, with a multi-byte
8446        // character mixed in so the truncation is exercised on `chars()`
8447        // counting rather than bytes.
8448        let long_line = format!("fix the thing 🎉 {}", "x".repeat(400));
8449        let title = pr_title(&long_line);
8450
8451        assert!(
8452            title.chars().count() <= PR_TITLE_MAX,
8453            "title must stay within PR_TITLE_MAX: {title:?} ({} chars)",
8454            title.chars().count()
8455        );
8456        assert!(
8457            title.chars().count() < 256,
8458            "title must stay within GitHub's 256-character limit: {title:?}"
8459        );
8460        assert!(
8461            title.ends_with('…'),
8462            "a truncated title must say so: {title:?}"
8463        );
8464    }
8465
8466    #[test]
8467    fn pr_title_leaves_a_short_title_untouched() {
8468        let title = pr_title("add retries\n\nmore detail below");
8469        assert_eq!(title, "add retries");
8470    }
8471
8472    #[test]
8473    fn pr_title_strips_markdown_heading_markers() {
8474        let title = pr_title("# Rework the config loader\n\ndetails");
8475        assert_eq!(title, "Rework the config loader");
8476    }
8477
8478    #[test]
8479    fn pr_title_of_pr_body_stays_within_githubs_limit() {
8480        let state = RunState::new(
8481            PathBuf::from("/repo"),
8482            "main".to_owned(),
8483            "abc1234".to_owned(),
8484            format!("fix the thing 🎉 {}", "x".repeat(400)),
8485            Config::default(),
8486        );
8487        let body = pr_body(&state, 'A');
8488        let title = pr_title(&body);
8489
8490        assert!(
8491            title.chars().count() < 256,
8492            "the title gh_pr_create sends must stay within GitHub's limit: {title:?}"
8493        );
8494    }
8495
8496    #[test]
8497    fn manual_merge_command_matches_the_configured_style() {
8498        let repo = Path::new("/repo");
8499        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
8500
8501        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
8502        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
8503
8504        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
8505        assert_eq!(
8506            squash,
8507            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
8508             \"Merge magi run 0832 (candidate A)\""
8509        );
8510
8511        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
8512        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
8513    }
8514
8515    #[test]
8516    fn a_nudge_gets_a_quarter_of_the_budget() {
8517        // The judge and implement budgets magi ships with.
8518        assert_eq!(retry_budget(secs(1200), true), secs(300));
8519        assert_eq!(retry_budget(secs(3600), true), secs(900));
8520    }
8521
8522    #[test]
8523    fn a_resent_prompt_keeps_the_whole_budget() {
8524        // The seat kept no context, so the retry is the original job again and
8525        // shortening it would only guarantee a second failure.
8526        assert_eq!(retry_budget(secs(1200), false), secs(1200));
8527        assert_eq!(retry_budget(secs(60), false), secs(60));
8528    }
8529
8530    #[test]
8531    fn the_floor_never_exceeds_the_original_budget() {
8532        // A short configured timeout must not be *raised* by the floor: the
8533        // operator asked for a bound, and a retry may not outlast the attempt
8534        // it is retrying.
8535        assert_eq!(retry_budget(secs(60), true), secs(60));
8536        assert_eq!(retry_budget(secs(480), true), secs(120));
8537        assert_eq!(retry_budget(secs(0), true), secs(0));
8538    }
8539
8540    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
8541        agent::CommandEvidence {
8542            id: "item1".to_owned(),
8543            description: "cargo test".to_owned(),
8544            exit_code,
8545            result_summary: String::new(),
8546            source: "codex".to_owned(),
8547        }
8548    }
8549
8550    #[test]
8551    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
8552        // No evidence is not the same fact as unconfirmed evidence: a
8553        // backend with no adapter, or a reply that ran no commands at all,
8554        // must not be misread as carrying a dangling job.
8555        assert!(!has_unconfirmed_command(&[]));
8556    }
8557
8558    #[test]
8559    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
8560        // Deliberately not a check on the exit code's *value*: a fixer
8561        // legitimately runs something that fails mid-iteration before it
8562        // succeeds, and that must never by itself reopen a valid report.
8563        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
8564        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
8565        assert!(!has_unconfirmed_command(&[
8566            evidence(Some(0)),
8567            evidence(Some(101))
8568        ]));
8569    }
8570
8571    #[test]
8572    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
8573        assert!(has_unconfirmed_command(&[
8574            evidence(Some(0)),
8575            evidence(None)
8576        ]));
8577    }
8578
8579    #[test]
8580    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
8581        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
8582        assert_eq!(
8583            verified_noop_claim(true, &[], text).as_deref(),
8584            Some("already fixed by b32cfc4, on main.")
8585        );
8586    }
8587
8588    #[test]
8589    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
8590        // A timeout or a bad exit code reads as the ordinary loss it is,
8591        // whatever the reply's own prose claims.
8592        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
8593        assert!(verified_noop_claim(false, &[], text).is_none());
8594    }
8595
8596    #[test]
8597    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
8598        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
8599        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
8600        // A confirmed command alongside the marker is fine.
8601        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
8602    }
8603
8604    #[test]
8605    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
8606        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
8607    }
8608
8609    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
8610    /// pair, in order, labelled A, B, C, ...
8611    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
8612        runner.state.candidates = shape
8613            .iter()
8614            .enumerate()
8615            .map(|(i, &(empty, verified))| Candidate {
8616                index: i,
8617                label: (b'A' + i as u8) as char,
8618                agent: "sonnet".to_owned(),
8619                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
8620                worktree: PathBuf::from(format!("/wt/{i}")),
8621                summary: String::new(),
8622                stat: String::new(),
8623                files: 0,
8624                commits: 0,
8625                empty,
8626                failed: None,
8627                verified_noop: verified.map(str::to_owned),
8628                duration_ms: 0,
8629                folded: false,
8630            })
8631            .collect();
8632    }
8633
8634    #[test]
8635    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
8636        ask_test_home();
8637        let mut runner = runner_at(RunStatus::Implementing);
8638        set_candidates(
8639            &mut runner,
8640            &[
8641                (true, Some("already on main at b32cfc4")),
8642                (true, Some("same fix, see the existing test")),
8643            ],
8644        );
8645
8646        runner
8647            .after_implement()
8648            .expect("a verified no-op is not an error");
8649
8650        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
8651    }
8652
8653    #[test]
8654    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
8655        ask_test_home();
8656        let mut runner = runner_at(RunStatus::Implementing);
8657        // Candidate A declares a verified no-op; candidate B simply wrote
8658        // nothing and said nothing about why. One candidate's claim is not
8659        // the whole run's agreement.
8660        set_candidates(
8661            &mut runner,
8662            &[(true, Some("already on main at b32cfc4")), (true, None)],
8663        );
8664
8665        let err = runner
8666            .after_implement()
8667            .expect_err("an unverified empty candidate must still fail the run");
8668
8669        assert!(
8670            err.to_string().contains("no candidate produced a change"),
8671            "{err}"
8672        );
8673        assert_eq!(runner.state.status, RunStatus::Failed);
8674    }
8675
8676    #[test]
8677    fn after_implement_still_fails_an_ordinary_all_empty_run() {
8678        ask_test_home();
8679        let mut runner = runner_at(RunStatus::Implementing);
8680        set_candidates(&mut runner, &[(true, None), (true, None)]);
8681
8682        let err = runner
8683            .after_implement()
8684            .expect_err("no candidate declared anything; this is an ordinary failure");
8685
8686        assert!(
8687            err.to_string().contains("no candidate produced a change"),
8688            "{err}"
8689        );
8690        assert_eq!(runner.state.status, RunStatus::Failed);
8691    }
8692}