Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::git;
39use crate::land;
40use crate::proc::Quiet as _;
41use crate::prompt::{
42    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
43};
44use crate::queue;
45use crate::run::{
46    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
47    DeliberationRound, DeliberationTurn, E2eStatus, FixRecord, GateFixRecord, JobRecord, JobStatus,
48    Judgement, MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, QuotaLoss,
49    ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, Tally, VoteRecord, tail,
50    write_artifact,
51};
52use crate::verdict::{
53    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
54    ReviewVote, Severity,
55};
56
57/// How much verification output is kept and fed back to the fixer.
58const OUTPUT_TAIL: usize = 8_000;
59
60/// Bytes of a failing command's output kept in an event, so the reason a run
61/// stopped is readable from the report without opening `run.json`.
62const EVENT_OUTPUT_TAIL: usize = 2_000;
63
64/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
65/// command's pid before releasing the build cache's lease.
66const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
67
68/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
69/// command's pid to actually exit before giving up and releasing anyway.
70///
71/// A timeout means the process was asked to die (`kill_on_drop`,
72/// `start_kill`), not that it already has — on Windows in particular that can
73/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
74/// the instant the command returns would let the very next acquirer (this
75/// run's own next round, another run's verification, the janitor's prune)
76/// start touching the same directory while it might still be writing to it,
77/// so this polls the actual pid — real confirmation, not a fixed guess —
78/// until it is gone or this ceiling is reached. It is still not full
79/// process-tree reaping: a grandchild the timed-out process spawned and that
80/// outlives it independently is invisible to a pid check, and continuing to
81/// observe and collect *that* stays a different piece of work with its own
82/// owner. Set generously because the common case returns early the moment
83/// the pid is confirmed gone, not because every timeout pays this in full.
84const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
85
86/// Consecutive review rounds with no tree progress (see
87/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
88/// instead of spending the rest of the round budget.
89///
90/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
91/// legitimately finds nothing left to change (its previous round's fix already
92/// covered it, and this round's reviewers re-raised only nits) looks the same
93/// as one that is spinning, for exactly one round. Two in a row is where the
94/// two stop being distinguishable, and a review round on this workload has
95/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
96/// third attempt at a tree that has not moved twice running is pure cost.
97/// This does not touch `review_rounds` itself, which stays the operator's
98/// call.
99pub(crate) const STAGNANT_LIMIT: usize = 2;
100
101/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
102/// a base that moved before giving up and leaving the run `Blocked` for a
103/// person.
104///
105/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
106/// that keeps moving faster than a run can catch it is not something more
107/// rebasing fixes, it is a person's call. Not the same *number as*
108/// `land_rounds` - this budget is spent before a pull request exists, land's
109/// after - but bounded for the identical reason, so it uses the same
110/// default. Counted across both call sites in [`Runner::finish_after_tally`]
111/// (once before review, once before the gate), because either one finding
112/// the base still moving is the same signal.
113const BASE_SYNC_ROUNDS: usize = 4;
114
115/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
116/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
117/// reply held no [`FixReport`].
118///
119/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
120/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
121/// "I'll pause here until the `cargo make check` background run reports
122/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
123/// No `FixReport` was ever collected from that seat, and the run moved on to
124/// the next review round regardless.
125///
126/// Bounded independently of `review_rounds` and `graph.retries`: this
127/// recovers one seat's missing report mid-round, not a new round of review or
128/// an ordinary parse retry, and must not itself become the unbounded wait the
129/// rest of this module exists to avoid.
130const MAX_FIX_CONTINUATIONS: usize = 2;
131
132/// One queued agent invocation.
133///
134/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
135/// CLI hung up on its own stream is asked again from the same job rather than
136/// rebuilt from scratch. See [`Runner::resume_undelivered`].
137#[derive(Clone)]
138struct SeatJob {
139    spec: AgentSpec,
140    seat: SeatState,
141    cwd: PathBuf,
142    prompt: String,
143    timeout: Duration,
144    allow_write: bool,
145    sessions: bool,
146    artifacts: PathBuf,
147    stem: String,
148}
149
150/// How the graph reads one agent invocation.
151///
152/// Quota is split out from an ordinary failure on purpose: a rate-limited call
153/// is known to fail again if retried now, so the retry loop must not spend an
154/// attempt on it. `Dropped` is split out for the opposite reason: unlike
155/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
156/// error JSON, never the agent's answer — a caller that matched only
157/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
158/// left to read that JSON as if it were usable output. `resume_undelivered`
159/// is the only caller that acts on it; everywhere else it is reported like an
160/// ordinary failure.
161enum AgentOutcome {
162    /// A usable output.
163    Ok(AgentOutput),
164    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
165    Quota(AgentOutput),
166    /// The CLI hung up on its own stream after billed work. See
167    /// [`agent::AgentOutput::work_undelivered`].
168    Dropped(AgentOutput),
169    /// Any other failure: a timeout, a bad exit code, an empty reply.
170    Failed(String),
171}
172
173/// A request to park the run at its next node boundary.
174///
175/// Cloning is how the request travels: the loop keeps one handle and hands a
176/// clone to each [`Runner`], and every clone points at the same flag. There
177/// is no channel because there is nothing to send - the only message is
178/// "park", it is idempotent, and a flag cannot be missed by a receiver that
179/// was not listening yet.
180///
181/// The boundary is what makes this cheap. Every node writes the run's state
182/// before the next one starts, and every node skips what is already recorded:
183/// `prep` returns early once candidates exist, `implement` asks only the seats
184/// with nothing on disk, `judge` returns early once judgements exist. So a
185/// parked run resumes into exactly the node it stopped before, and no agent
186/// work is thrown away. Killing the process mid-node, by contrast, loses
187/// whatever the seats in flight had not yet written - which for an implement
188/// wave is an hour of paid work.
189///
190/// A [`Runner`] watches two independent handles of this type - see
191/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
192/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
193/// clone covering the whole daemon's lifetime and is never asked to un-park,
194/// which is correct exactly because nothing is dispatched after it fires.
195/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
196/// that parks for an interrupted task must go on to run other tasks
197/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
198/// reusing the daemon-wide one.
199#[derive(Debug, Clone, Default)]
200pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
201
202impl Pause {
203    /// A pause nobody has asked for yet.
204    #[must_use]
205    pub fn new() -> Self {
206        Self::default()
207    }
208
209    /// Ask the run to park at its next node boundary. Idempotent.
210    pub fn park(&self) {
211        self.0.store(true, Ordering::SeqCst);
212    }
213
214    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
215    /// fold into the run's own `park` event - so an operator reading the run
216    /// later knows this was a deliberate interrupt rather than a shutdown or
217    /// a binary swap. The first reason recorded wins; a park already in
218    /// flight is not relabelled by a second, unrelated request.
219    pub fn park_because(&self, reason: impl Into<String>) {
220        let mut reason_guard = self
221            .1
222            .lock()
223            .unwrap_or_else(std::sync::PoisonError::into_inner);
224        if reason_guard.is_none() {
225            *reason_guard = Some(reason.into());
226        }
227        drop(reason_guard);
228        self.park();
229    }
230
231    /// Has a park been asked for?
232    #[must_use]
233    pub fn parked(&self) -> bool {
234        self.0.load(Ordering::SeqCst)
235    }
236
237    /// Why the park was asked for, when the caller used [`Pause::park_because`].
238    #[must_use]
239    pub fn reason(&self) -> Option<String> {
240        self.1
241            .lock()
242            .unwrap_or_else(std::sync::PoisonError::into_inner)
243            .clone()
244    }
245}
246
247/// Drives one run.
248pub struct Runner {
249    /// Run state; public so the CLI can report on it.
250    pub state: RunState,
251    roles: ResolvedRoles,
252    sem: Arc<Semaphore>,
253    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
254    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
255    /// this is never the same handle as `interrupt`.
256    pause: Pause,
257    /// Set when `magi serve`'s interrupt scheduler wants this specific run
258    /// parked at its next node boundary, to let a task marked
259    /// [`crate::queue::Task::interrupt`] run alone before this one carries
260    /// on. Unlike `pause`, a fresh, unshared handle per run - see
261    /// [`Runner::watch_interrupt`].
262    interrupt: Pause,
263}
264
265/// The commit a run branches from: the base branch as the remote has it.
266///
267/// Two failures this replaces. A run used to branch off `HEAD` and so refused
268/// to start on a dirty tree, which made `magi serve` decline every task for as
269/// long as the operator had work in progress - most of the time. Branching off
270/// the *local* base branch fixed that and introduced a worse one: `land` merges
271/// the winner on GitHub, nothing updates the local ref, and the next run
272/// branches off a base missing everything the previous runs landed. Two tasks
273/// in a row from a phone would have had the second silently re-implementing
274/// against stale code and opening a pull request that reverted the first.
275///
276/// Only refs move here - no checkout, no local branch, no merge - so it is safe
277/// with uncommitted work in the tree. A machine with no network still starts:
278/// the fetch may fail and the local tip is used with a warning, because
279/// refusing to run offline is a worse failure than running against a base the
280/// operator can see for themselves.
281///
282/// One function, called by both entry points. Two answers to "where does a run
283/// branch from" is the kind of drift nobody notices until a diff is wrong.
284async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
285    let tracking = format!("{remote}/{base_branch}");
286    let fetched = git::fetch(repo, remote, base_branch).await;
287    if let Ok(out) = &fetched
288        && out.ok()
289        && git::rev_exists(repo, &tracking).await
290    {
291        return git::rev_parse(repo, &tracking).await;
292    }
293    let why = match &fetched {
294        Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
295        Ok(_) => format!("{remote} has no {base_branch}"),
296        Err(e) => e.to_string(),
297    };
298    tracing::warn!(
299        "could not read {tracking} ({why}); branching off the local \
300         {base_branch} instead, which may be behind"
301    );
302    git::rev_parse(repo, base_branch).await.with_context(|| {
303        format!(
304            "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
305             branch that exists"
306        )
307    })
308}
309
310/// Exclusive claim on one run's `magi fix` step, released on drop — including
311/// on an early return or a panic.
312///
313/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
314/// manual `magi fix` invocations against the same run are otherwise
315/// invisible to each other and would race to remove and recreate the same
316/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
317/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
318/// which is only ever reclaimed later, out of band, by
319/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
320/// `magi fix` invocation is not necessarily running under either of those, so
321/// nothing would ever sweep a lock a killed or crashed process left behind.
322/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
323/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
324/// lease use: an unreadable or unparsable pid, or a liveness query the
325/// platform cannot answer, reads as alive and the lock is left in place.
326struct FixClaim {
327    path: PathBuf,
328}
329
330impl FixClaim {
331    fn acquire(dir: &Path) -> Result<Self> {
332        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
333        let path = dir.join("fix.lock");
334        match Self::create(&path) {
335            Ok(claim) => Ok(claim),
336            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
337                if Self::reclaim_if_dead(&path) {
338                    Self::create(&path).with_context(|| format!("lock {}", path.display()))
339                } else {
340                    bail!(
341                        "another `magi fix` is already running for this run ({} exists)",
342                        path.display()
343                    )
344                }
345            }
346            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
347        }
348    }
349
350    fn create(path: &Path) -> std::io::Result<Self> {
351        let mut f = std::fs::OpenOptions::new()
352            .write(true)
353            .create_new(true)
354            .open(path)?;
355        use std::io::Write as _;
356        // Read back by `reclaim_if_dead` on a later, stuck invocation.
357        writeln!(f, "{}", std::process::id())?;
358        Ok(Self {
359            path: path.to_owned(),
360        })
361    }
362
363    /// True if the lock named a process confirmed dead, in which case it was
364    /// also removed. Never true on an unreadable file, an unparsable pid, or
365    /// a liveness query the platform cannot answer — see this type's own doc.
366    fn reclaim_if_dead(path: &Path) -> bool {
367        let dead = std::fs::read_to_string(path)
368            .ok()
369            .and_then(|body| body.trim().parse::<u32>().ok())
370            .is_some_and(|pid| !crate::proc::pid_alive(pid));
371        dead && std::fs::remove_file(path).is_ok()
372    }
373}
374
375impl Drop for FixClaim {
376    fn drop(&mut self) {
377        let _ = std::fs::remove_file(&self.path);
378    }
379}
380
381impl Runner {
382    /// Start a fresh run against `repo`.
383    pub async fn start(repo: &Path, instruction: String, config: Config) -> Result<Self> {
384        let repo = git::toplevel(repo).await?;
385        let missing = agent::missing_programs(&config.agents);
386        if !missing.is_empty() {
387            bail!(
388                "these agent programs are not on PATH: {}. Fix the roster in \
389                 magi.toml or install them.",
390                missing.join(", ")
391            );
392        }
393        let base_branch = match config.merge.base.clone() {
394            Some(b) => b,
395            None => git::current_branch(&repo)
396                .await?
397                .context("HEAD is detached; set [merge] base in magi.toml")?,
398        };
399        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
400        // Still worth saying out loud. The operator's uncommitted work is not
401        // part of this run, and someone watching a candidate fail to use a
402        // change they just made deserves to know why.
403        if !git::is_clean(&repo).await? {
404            tracing::warn!(
405                "{} has uncommitted changes; they are not part of this run, \
406                 which branches off {base_branch} ({})",
407                repo.display(),
408                &base_commit[..base_commit.len().min(8)]
409            );
410        }
411        let roles = config.resolve_roles()?;
412        let max_parallel = config.graph.max_parallel.max(1);
413        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
414        state.event("start", format!("run {} created", state.id));
415        state.save()?;
416        Ok(Self {
417            state,
418            roles,
419            sem: Arc::new(Semaphore::new(max_parallel)),
420            pause: Pause::new(),
421            interrupt: Pause::new(),
422        })
423    }
424
425    /// Open a review-only run against work that already exists on `branch`.
426    ///
427    /// The expensive half of the graph is the implement wave — measured at
428    /// 111 and 134 internal tool-loop turns on this repository, against a
429    /// handful for a judge or a reviewer. The cheap half is worth running on
430    /// hand-written work too, and there was no way to reach it.
431    ///
432    /// No new state and no schema change are needed: a run with **one** viable
433    /// candidate and a tally already decided degrades `execute` to exactly
434    /// review → gate → merge, because `judge` skips a single-candidate field,
435    /// `deliberate` has fewer than two first choices to reconcile, `vote`
436    /// returns early, `tally` is already present and `fold_losers` has no
437    /// losers. Resuming such a run therefore does the right thing as well.
438    pub async fn review(repo: &Path, branch: &str, config: Config) -> Result<Self> {
439        let repo = git::toplevel(repo).await?;
440        let missing = agent::missing_programs(&config.agents);
441        if !missing.is_empty() {
442            bail!(
443                "these agent programs are not on PATH: {}. Fix the roster in \
444                 magi.toml or install them.",
445                missing.join(", ")
446            );
447        }
448        if !git::branch_exists(&repo, branch).await? {
449            bail!("no branch `{branch}` in {}", repo.display());
450        }
451        let base_branch = match config.merge.base.clone() {
452            Some(b) => b,
453            None => git::current_branch(&repo)
454                .await?
455                .context("HEAD is detached; set [merge] base in magi.toml")?,
456        };
457        if base_branch == branch {
458            bail!("`{branch}` is the base branch; there is nothing to review against");
459        }
460        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
461
462        let roles = config.resolve_roles()?;
463        let max_parallel = config.graph.max_parallel.max(1);
464        // The commit subjects are the closest thing to a task statement that
465        // existing work carries, and the reviewers are told as much.
466        let log = git::log_oneline(&repo, &base_commit, branch)
467            .await
468            .unwrap_or_default();
469        let instruction = format!(
470            "Review the work already on branch `{branch}`. There is no task \
471             statement: what the change claims to do is whatever its commits \
472             say.\n\n{}",
473            if log.trim().is_empty() {
474                "(no commit messages)"
475            } else {
476                log.trim()
477            }
478        );
479        let mut state = RunState::new(
480            repo.clone(),
481            base_branch,
482            base_commit.clone(),
483            instruction,
484            config,
485        );
486
487        // An attached worktree, so the fixer's commits land on the branch under
488        // review rather than on a detached head nobody will look at again.
489        let worktree = state.worktree_root().join("under-review");
490        if let Some(parent) = worktree.parent() {
491            tokio::fs::create_dir_all(parent).await.ok();
492        }
493        let path = worktree.to_string_lossy().to_string();
494        git::git(&repo, &["worktree", "add", &path, branch])
495            .await
496            .with_context(|| {
497                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
498            })?;
499
500        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
501            .await
502            .unwrap_or(0);
503        if commits == 0 {
504            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
505        }
506        let files = git::changed_files(&worktree, &base_commit, "HEAD")
507            .await
508            .map(|f| f.len())
509            .unwrap_or(0);
510        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
511            .await
512            .unwrap_or_default();
513
514        state.candidates.push(Candidate {
515            index: 0,
516            label: 'A',
517            // Not an agent id on purpose: nothing in the roster wrote this, and
518            // the stats tables must not credit anyone with a win for it.
519            agent: "(existing branch)".to_owned(),
520            branch: branch.to_owned(),
521            worktree,
522            summary: String::new(),
523            stat,
524            files,
525            commits,
526            empty: false,
527            failed: None,
528            verified_noop: None,
529            duration_ms: 0,
530            folded: false,
531        });
532        state.tally = Some(Tally {
533            first_choice: BTreeMap::from([('A', 0)]),
534            borda: BTreeMap::new(),
535            winner: 'A',
536            rankings: 0,
537            unanimous_initial: false,
538            deliberated: false,
539            changed_votes: 0,
540            unanimous_final: false,
541            tie_break: None,
542            // No panel sat, so no quorum applies. Zero judges is the correct
543            // number for work that never competed, and must not be reported as
544            // a collapsed panel.
545            judges: 0,
546            present: 0,
547            quorum: 0,
548            met_quorum: true,
549            uncontested: Some("review-only run: nothing competed".to_owned()),
550        });
551        state.status = RunStatus::Reviewing;
552        state.event(
553            "start",
554            format!(
555                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
556                state.id
557            ),
558        );
559        state.save()?;
560        Ok(Self {
561            state,
562            roles,
563            sem: Arc::new(Semaphore::new(max_parallel)),
564            pause: Pause::new(),
565            interrupt: Pause::new(),
566        })
567    }
568
569    /// Reopen an existing run.
570    pub fn resume(id: &str) -> Result<Self> {
571        let state = RunState::load(id)?;
572        let roles = state.config.resolve_roles()?;
573        let max_parallel = state.config.graph.max_parallel.max(1);
574        Ok(Self {
575            state,
576            roles,
577            sem: Arc::new(Semaphore::new(max_parallel)),
578            pause: Pause::new(),
579            interrupt: Pause::new(),
580        })
581    }
582
583    /// Walk the graph to a terminal state, skipping nodes already recorded.
584    ///
585    /// Every way a run is driven - the queue loop, `magi run`, a resume from
586    /// the phone - ends here, so this is the one place a run that ended
587    /// Blocked / Stalled / Failed, or died with an error, is announced to the
588    /// notification centre. Best-effort: see [`crate::notices::raise`].
589    pub async fn execute(&mut self) -> Result<()> {
590        let result = self.execute_graph().await;
591        let ended = if result.is_err() {
592            Some(crate::notices::run_stopped(&self.state.id, &self.state))
593        } else {
594            crate::notices::run_ended(&self.state)
595        };
596        if let Some(notice) = ended {
597            crate::notices::raise(notice);
598        }
599        result
600    }
601
602    async fn execute_graph(&mut self) -> Result<()> {
603        // Moving again, so it is no longer parked. Set before the walk rather
604        // than in `resume`, so every way of re-entering the graph clears it
605        // and a card cannot claim a run is waiting to be resumed while the
606        // agents are already working.
607        self.state.parked = false;
608        // Any seat this state still lists as answering belongs to whatever
609        // process last drove this run — this one included, if it crashed
610        // mid-wave. Cleared and flushed immediately, before anything else
611        // runs, so a resume can never show a seat as live when nothing is
612        // asking it anything yet; the node that actually dispatches the next
613        // wave repopulates it.
614        self.state.clear_active();
615        // Recorded in the same spot, and flushed together with the clear
616        // above: this is the pid a reader checks (`RunState::liveness`) when
617        // no daemon claim exists to answer "is a process still driving this
618        // run" — a plain `magi run` / `magi review` typed into a terminal
619        // claims nothing there. Always overwritten, never only-if-absent, so
620        // a resumed run's stale pid from a previous, possibly-dead process
621        // can never survive into this one's own report. Unlike
622        // `clear_active`, this changes on every single `execute()` call, so
623        // the save below is now unconditional rather than only-if-cleared.
624        //
625        // `driver_started_at` is recorded in the same breath, from this same
626        // pid, so `liveness` can tell a live pid that is genuinely still us
627        // apart from one the OS has since handed to an unrelated process —
628        // see that field's own doc for why the pid alone is not enough.
629        let pid = std::process::id();
630        self.state.driver_pid = Some(pid);
631        self.state.driver_started_at = crate::proc::process_started_at(pid);
632        self.state.save()?;
633        // A run that already lost its quorum never resumes into the verdict
634        // machinery: `deliberate` and `vote` would otherwise clobber the
635        // stalled marker back to Voting and the run would keep going past a
636        // verdict that is no longer trustworthy. Everything already recorded is
637        // kept, so the run stays resumable (or foldable) for a human to pick up.
638        //
639        // On --resume the run gets one chance to repair itself: the seats a
640        // rate limit took out are re-asked. If their quota has since reset and
641        // the quorum is restored, the run picks up and finishes; otherwise it
642        // stays stale and still-resumable for a later retry. If it does not
643        // recover, the returned status stays `Stalled` and nothing was
644        // clobbered (the recovery only mutates entries for the lost seats).
645        if self.state.status == RunStatus::Stalled {
646            if self.recover_stall().await? {
647                self.finish_after_tally().await?;
648            } else {
649                // Still below quorum: persist the marker and stay resumable.
650                self.state.save()?;
651            }
652            return Ok(());
653        }
654        // A run parked inside `land` - watching CI, mid fix-round, or
655        // waiting on the owner's merge approval - resumes directly into it,
656        // never back through `prep`. Everything before `merge` already
657        // concluded; that is the only way `status` reaches `Landing` in the
658        // first place. Re-walking `review_loop` first would also be actively
659        // wrong: its own status recomputation (see its doc) treats any
660        // clean round as reason to set `status` to `Gating`, which would
661        // clobber this marker before `merge` ever ran, and this run would
662        // never find its way back into `land` at all.
663        if self.state.status == RunStatus::Landing {
664            self.run_land().await?;
665            // `run_land` may have settled the run right here - CI came back
666            // green and the PR merged, say - without ever passing back
667            // through `merge`'s own trailing call. Whatever it left `status`
668            // as is what this has to read.
669            self.settle_questions();
670            return Ok(());
671        }
672        self.prep().await?;
673        if self.park_here()? {
674            return Ok(());
675        }
676        self.advise().await?;
677        if self.park_here()? {
678            return Ok(());
679        }
680        self.implement().await?;
681        if self.park_here()? {
682            return Ok(());
683        }
684        // `after_implement` already saved the state and settled any open
685        // questions when it set this; nothing later in the graph has
686        // anything to judge.
687        if self.state.status == RunStatus::VerifiedNoop {
688            return Ok(());
689        }
690        self.judge().await?;
691        if self.park_here()? {
692            return Ok(());
693        }
694        self.deliberate().await?;
695        if self.park_here()? {
696            return Ok(());
697        }
698        self.vote().await?;
699        if self.park_here()? {
700            return Ok(());
701        }
702        self.tally()?;
703        // A verdict that lost its quorum is not trustworthy: do not review,
704        // gate, or merge on it. Everything already done is kept, so the run
705        // stays resumable (or foldable); the human can replace the agent that
706        // ran out of quota and pick it up.
707        if self.state.status == RunStatus::Stalled {
708            // Persist the stalled marker now — the normal end-of-execute save
709            // below is below this early return, and without it a resumed run
710            // would reload a pre-tally status and keep going.
711            self.state.save()?;
712            return Ok(());
713        }
714        self.finish_after_tally().await?;
715        Ok(())
716    }
717
718    /// Park here if asked to, recording it in the run's own timeline.
719    ///
720    /// Returns whether the caller should stop walking the graph. The state is
721    /// saved either way by the node that just finished; this adds the event so
722    /// the operator's card says why a run that is neither finished nor moving
723    /// is sitting where it is.
724    fn park_here(&mut self) -> Result<bool> {
725        // Either handle asking is enough - see `Pause`'s own doc for why
726        // they are never the same one. `interrupt` is checked second so a
727        // reason it carries is preferred in the message below over a plain
728        // shutdown park racing it at the same boundary.
729        if !self.pause.parked() && !self.interrupt.parked() {
730            return Ok(false);
731        }
732        let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
733            Some(reason) => format!(
734                "parked after `{}` ({reason}) — resume to carry on from here",
735                self.state.status.as_str()
736            ),
737            None => format!(
738                "parked after `{}` — resume to carry on from here",
739                self.state.status.as_str()
740            ),
741        };
742        self.state.event("park", why);
743        self.state.parked = true;
744        self.state.save()?;
745        Ok(true)
746    }
747
748    /// Hand the runner the pause `magi serve`'s own shutdown watches.
749    pub fn on_pause(&mut self, pause: Pause) {
750        self.pause = pause;
751    }
752
753    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
754    /// scheduler asking this one run - and no other - to park so a task
755    /// marked [`crate::queue::Task::interrupt`] can run alone. See
756    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
757    /// handle.
758    pub fn watch_interrupt(&mut self, pause: Pause) {
759        self.interrupt = pause;
760    }
761
762    /// Abandon this run's own open questions, once `status` has actually
763    /// settled rather than merely paused.
764    ///
765    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
766    /// either back up with the candidates, the review round and the seat
767    /// sessions already on disk, so a question an implementer asked mid-round
768    /// may still get a real answer read by a real resume. Only the statuses
769    /// `resumable` excludes are actually final: the run merged, it reached
770    /// `Ready` with nothing left to do, it failed outright with no
771    /// established point to continue from, or every candidate agreed, with
772    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
773    /// every one of those the seat that asked is gone for good, exactly like
774    /// the run being deleted under `magi run rm` - so the same cleanup
775    /// applies, worded for what actually happened instead of "the run was
776    /// deleted".
777    ///
778    /// Best-effort and silent on success: called from every place `status`
779    /// can land on one of those three, including ones a resumed run revisits,
780    /// so it must cost nothing when there was nothing open to begin with.
781    fn settle_questions(&mut self) {
782        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
783            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
784        }
785    }
786
787    /// The tail of the graph after a trustworthy tally: fold losers, review,
788    /// gate, merge, and persist.
789    async fn finish_after_tally(&mut self) -> Result<()> {
790        self.fold_losers().await?;
791        // Before review starts, and again right before the gate: a run's
792        // review rounds can themselves take long enough for the base to move
793        // a second time, and the gate is the one node whose "green" gets
794        // acted on.
795        self.sync_to_base().await?;
796        self.review_loop().await?;
797        self.sync_to_base().await?;
798        self.gate().await?;
799        self.merge().await?;
800        self.state.save()?;
801        Ok(())
802    }
803
804    // ---------------------------------------------------------------- prep
805
806    async fn prep(&mut self) -> Result<()> {
807        if !self.state.candidates.is_empty() {
808            return Ok(());
809        }
810        self.state.status = RunStatus::Prep;
811        let repo = self.state.repo.clone();
812        let base = self.state.base_commit.clone();
813        let root = self.state.worktree_root();
814        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
815
816        // The hook is the write-time half of the blindness contract; the
817        // presentation filter in `blind` is the half that cannot be bypassed.
818        let hooks_dir = self.state.dir().join("hooks");
819        if self.state.config.blind.commit_msg_hook {
820            std::fs::create_dir_all(&hooks_dir)
821                .with_context(|| format!("create {}", hooks_dir.display()))?;
822            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
823            let path = hooks_dir.join("commit-msg");
824            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
825            make_executable(&path)?;
826            // Ref-counted rather than a plain idempotent set: with more than
827            // one run able to be in flight in the same repository at once
828            // (see `Config::daemon.max_concurrent_runs`), a bare "already
829            // true?" check cannot tell "another run of mine still needs
830            // this" from "nobody does", and the run that happens to finish
831            // first would disable the hook out from under a sibling still
832            // relying on it.
833            git::acquire_worktree_config(&repo).await?;
834            self.state.enabled_worktree_config = true;
835        }
836
837        for (index, (spec, label)) in self
838            .roles
839            .implementers
840            .clone()
841            .into_iter()
842            .zip(labels)
843            .enumerate()
844        {
845            let branch = self.state.branch_for(label);
846            let worktree = root.join(format!("cand-{label}"));
847            git::worktree_add_branch(&repo, &worktree, &branch, &base).await?;
848            if self.state.config.blind.commit_msg_hook {
849                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
850            }
851            git::local_exclude(&worktree, "/.magi/").await?;
852            self.state.candidates.push(Candidate {
853                index,
854                label,
855                agent: spec.id.clone(),
856                branch,
857                worktree,
858                summary: String::new(),
859                stat: String::new(),
860                files: 0,
861                commits: 0,
862                empty: false,
863                failed: None,
864                verified_noop: None,
865                duration_ms: 0,
866                folded: false,
867            });
868        }
869
870        for j in 1..=self.roles.judges.len() {
871            let wt = root.join(format!("judge-{j}"));
872            if !wt.exists() {
873                git::worktree_add_detached(&repo, &wt, &base).await?;
874            }
875        }
876
877        // Disposable, detached checkouts for the design-deliberation stage's
878        // advisor seats — the same shape as the judges' above, at the same
879        // base commit, since advisors also only ever read. Sized off the
880        // configured count directly rather than a resolved roster: unlike
881        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
882        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
883        // `prep` has no `ResolvedRoles` field to read a count from here.
884        if self.state.config.graph.advise {
885            for k in 1..=self.state.config.graph.advisors {
886                let wt = root.join(format!("advisor-{k}"));
887                if !wt.exists() {
888                    git::worktree_add_detached(&repo, &wt, &base).await?;
889                }
890            }
891        }
892
893        // A judge cannot tell it is looking at its own patch — the seats keep
894        // separate conversations — but a panel that shares agents with the
895        // field is less independent than it looks, and that is worth saying out
896        // loud once per run rather than leaving it in the config.
897        let authors: Vec<&str> = self
898            .roles
899            .implementers
900            .iter()
901            .map(|a| a.id.as_str())
902            .collect();
903        let overlap: Vec<String> = self
904            .roles
905            .judges
906            .iter()
907            .enumerate()
908            .filter(|(_, j)| authors.contains(&j.id.as_str()))
909            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
910            .collect();
911        if !overlap.is_empty() {
912            let note = format!(
913                "{} also authored a candidate; blind, but the panel is less \
914                 independent than {} distinct agents would be",
915                overlap.join(", "),
916                self.roles.judges.len()
917            );
918            self.state.event("prep", note);
919        }
920
921        self.state.event(
922            "prep",
923            format!(
924                "{} candidates, {} judges, base {} ({})",
925                self.state.candidates.len(),
926                self.roles.judges.len(),
927                &self.state.base_commit[..7.min(self.state.base_commit.len())],
928                self.state.base_branch
929            ),
930        );
931        self.state.status = RunStatus::Implementing;
932        self.state.save()?;
933        Ok(())
934    }
935
936    // -------------------------------------------------------------- advise
937
938    /// The design-deliberation stage: independent, read-only advisor seats
939    /// each sketch a design before any implementer touches the repository,
940    /// and (when at least one produced a usable proposal) a synthesis seat
941    /// blends them into a brief `implement` carries in every candidate's
942    /// prompt.
943    ///
944    /// `[graph] advise` is the on/off switch, on by default; `[graph]
945    /// advisors` is the proposal count. Everything here is best-effort and
946    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
947    /// that cannot reach quota, or a synthesis seat that produced nothing
948    /// usable all leave `implement` exactly as it was before this stage
949    /// existed — the task instruction alone — rather than failing the whole
950    /// competition over an enrichment stage. Every outcome is still recorded
951    /// as an event, so a run that got nothing from this stage says why.
952    ///
953    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
954    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
955    /// resumed run whose stage failed would re-run it, and re-spend the
956    /// agent calls, on every reentry before `implement`.
957    ///
958    /// Also skipped once any candidate shows implementation progress — the
959    /// exact predicate `implement` itself uses to decide a candidate is no
960    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
961    /// is not enough: a run created by an older binary that predates this
962    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
963    /// an already-`Implementing`-or-later run under this build would
964    /// otherwise walk straight back through `prep` (a no-op once candidates
965    /// exist) into this node and spawn every advisor seat against worktrees
966    /// `prep` never recreated — after implementation has already started,
967    /// which is exactly the invariant this stage exists to guarantee.
968    async fn advise(&mut self) -> Result<()> {
969        let implement_untouched = self
970            .state
971            .candidates
972            .iter()
973            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
974        if !self.state.config.graph.advise || self.state.advise_attempted {
975            return Ok(());
976        }
977        if !implement_untouched {
978            self.state.event(
979                "advise",
980                "skipping the design-deliberation stage: at least one \
981                 candidate already shows implementation progress, so this \
982                 run is past the point the stage exists to run before"
983                    .to_owned(),
984            );
985            self.state.advise_attempted = true;
986            self.state.save()?;
987            return Ok(());
988        }
989        let run_id = self.state.id.clone();
990        let prompts = self.state.config.prompts.clone();
991        let instruction = self.state.instruction.clone();
992        let language = self.state.config.graph.language.clone();
993        let root = self.state.worktree_root();
994        let n = self.state.config.graph.advisors;
995        let where_recorded = self.state.dir().join("run.json");
996
997        let seats = match self.state.config.advisors() {
998            Ok(seats) if !seats.is_empty() => seats,
999            Ok(_) => {
1000                self.state.event(
1001                    "advise",
1002                    format!(
1003                        "[graph] advisors is 0; skipping the design-deliberation \
1004                         stage and continuing without a synthesis brief (see {})",
1005                        where_recorded.display()
1006                    ),
1007                );
1008                self.state.advise_attempted = true;
1009                self.state.save()?;
1010                return Ok(());
1011            }
1012            Err(e) => {
1013                self.state.event(
1014                    "advise",
1015                    format!(
1016                        "could not resolve advisor seats ({e:#}); continuing \
1017                         without a design-deliberation brief (see {})",
1018                        where_recorded.display()
1019                    ),
1020                );
1021                self.state.advise_attempted = true;
1022                self.state.save()?;
1023                return Ok(());
1024            }
1025        };
1026
1027        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1028        let artifacts = agent::artifacts_dir(&self.state.dir());
1029        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1030
1031        let mut jobs = Vec::new();
1032        for (i, spec) in seats.iter().cloned().enumerate() {
1033            let seat_key = format!("advisor-{}", i + 1);
1034            let seat = self.seat(&seat_key, &spec.id);
1035            jobs.push(SeatJob {
1036                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1037                spec,
1038                seat,
1039                cwd: worktrees[i % worktrees.len()].clone(),
1040                timeout,
1041                allow_write: false,
1042                sessions: false,
1043                artifacts: artifacts.clone(),
1044                stem: seat_key,
1045            });
1046        }
1047
1048        self.state.event(
1049            "advise",
1050            format!(
1051                "{} advisor seat(s) sketching a design in parallel",
1052                jobs.len()
1053            ),
1054        );
1055        let mut quota_losses = Vec::new();
1056        let cache = self.state.config.cache_dir();
1057        let ctx = WaveCtx {
1058            run: &run_id,
1059            node: "advise",
1060            prompts: &prompts,
1061            cache: cache.as_deref(),
1062            round: None,
1063        };
1064        let results = ask_json_wave::<Proposal>(
1065            jobs,
1066            Arc::clone(&self.sem),
1067            self.state.config.graph.retries,
1068            &ctx,
1069            &mut quota_losses,
1070            &mut self.state,
1071            &|p: &Proposal| p.validate(),
1072        )
1073        .await;
1074        self.state.quota.extend(quota_losses);
1075
1076        let mut records = Vec::with_capacity(results.len());
1077        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1078            let agent_id = seat.agent.clone();
1079            self.state.seats.insert(seat.key.clone(), seat);
1080            match res {
1081                Ok((proposal, out)) => {
1082                    self.state
1083                        .event("advise", format!("advisor-{} proposed a design", i + 1));
1084                    records.push(advise::AdvisorRecord::proposed(
1085                        i + 1,
1086                        agent_id,
1087                        proposal,
1088                        out.duration_ms,
1089                    ));
1090                }
1091                Err(e) => {
1092                    self.state.event(
1093                        "advise",
1094                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1095                    );
1096                    records.push(advise::AdvisorRecord::failed(
1097                        i + 1,
1098                        agent_id,
1099                        e.to_string(),
1100                    ));
1101                }
1102            }
1103        }
1104
1105        let mut advice = advise::Advice {
1106            records,
1107            synthesis: None,
1108        };
1109        if advice.proposals().is_empty() {
1110            self.state.event(
1111                "advise",
1112                "no advisor produced a usable proposal; continuing without a \
1113                 synthesis brief"
1114                    .to_owned(),
1115            );
1116        } else {
1117            match self
1118                .synthesize_brief(
1119                    &advice,
1120                    &instruction,
1121                    &language,
1122                    &worktrees[0],
1123                    &artifacts,
1124                    &run_id,
1125                    &prompts,
1126                    cache.as_deref(),
1127                )
1128                .await
1129            {
1130                Ok(Some(text)) => {
1131                    self.state.event(
1132                        "advise",
1133                        "synthesized a design brief for the implementer".to_owned(),
1134                    );
1135                    advice.synthesis = Some(text);
1136                }
1137                Ok(None) => {
1138                    self.state.event(
1139                        "advise",
1140                        "the synthesis seat produced nothing usable; continuing \
1141                         without a design brief"
1142                            .to_owned(),
1143                    );
1144                }
1145                Err(e) => {
1146                    self.state.event(
1147                        "advise",
1148                        format!("could not synthesize a design brief: {e:#}"),
1149                    );
1150                }
1151            }
1152        }
1153        advise::apply_reflection(&mut advice);
1154
1155        self.state.advice = Some(advice);
1156        self.state.advise_attempted = true;
1157        self.state.save()?;
1158        Ok(())
1159    }
1160
1161    /// The synthesis seat: reads every advisor's proposal and blends them
1162    /// into the design brief `advise` stores on [`RunState::advice`]. Split
1163    /// out of [`Runner::advise`] only for readability — it is not called
1164    /// anywhere else.
1165    ///
1166    /// Picked the same way [`crate::talk`]'s standing conversation and
1167    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
1168    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
1169    /// order (a claude seat, else the first runnable agent in roster order)
1170    /// used when that field is unset — see `[roles] synthesizer`'s own doc
1171    /// in [`crate::config`] for why a dedicated field exists here at all.
1172    #[allow(clippy::too_many_arguments)]
1173    async fn synthesize_brief(
1174        &mut self,
1175        advice: &advise::Advice,
1176        instruction: &str,
1177        language: &str,
1178        cwd: &Path,
1179        artifacts: &Path,
1180        run_id: &str,
1181        prompts: &Prompts,
1182        cache: Option<&Path>,
1183    ) -> Result<Option<String>> {
1184        let want = self.state.config.roles.synthesizer.as_deref();
1185        let spec = agent::pick(&self.state.config.agents, want, &agent::installed)?;
1186        let mut seat = self.seat("advise-synthesis", &spec.id);
1187        let proposals = advice.proposals();
1188        let mut prompt = prompt::with_overlay(
1189            prompt::synthesize_brief(instruction, &proposals, language),
1190            prompts.overlay("advise"),
1191        );
1192        if cache.is_some() {
1193            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
1194            // below — see `prompt::build_cache_note`'s doc for why telling a
1195            // read-only seat to build through the shared cache is exactly how
1196            // a sandbox's write refusal gets misread as a defect.
1197            prompt.push('\n');
1198            prompt.push_str(&prompt::build_cache_note("advise", false));
1199        }
1200        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1201        let out = agent::invoke(
1202            &spec,
1203            &mut seat,
1204            &Invocation {
1205                cwd,
1206                prompt: &prompt,
1207                timeout,
1208                allow_write: false,
1209                sessions: false,
1210                artifacts,
1211                stem: "advise-synthesis",
1212                run: run_id,
1213                node: "advise",
1214                cache_dir: None,
1215                attachments: &[],
1216            },
1217        )
1218        .await?;
1219        self.state.seats.insert(seat.key.clone(), seat);
1220        if !out.usable() {
1221            return Ok(None);
1222        }
1223        let text =
1224            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1225        Ok((!text.trim().is_empty()).then_some(text))
1226    }
1227
1228    // ----------------------------------------------------------- implement
1229
1230    async fn implement(&mut self) -> Result<()> {
1231        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1232        // agent files with `magi task add` name the run that paid for it. The
1233        // prompt overlay is cloned alongside it because the waves borrow it
1234        // while `self` is mutably borrowed by the node's own bookkeeping.
1235        let run_id = self.state.id.clone();
1236        let prompts = self.state.config.prompts.clone();
1237        let todo: Vec<usize> = self
1238            .state
1239            .candidates
1240            .iter()
1241            .enumerate()
1242            .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1243            .map(|(i, _)| i)
1244            .collect();
1245        if todo.is_empty() {
1246            return self.after_implement();
1247        }
1248        self.state.status = RunStatus::Implementing;
1249
1250        let language = self.state.config.graph.language.clone();
1251        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1252        let sessions = self.state.config.graph.sessions;
1253        let artifacts = agent::artifacts_dir(&self.state.dir());
1254        // The design-deliberation stage's blended brief, when `advise` found
1255        // one — carried into every implementer's prompt the same way
1256        // regardless of which candidate it is.
1257        let brief = self
1258            .state
1259            .advice
1260            .as_ref()
1261            .and_then(|a| a.synthesis.as_deref())
1262            .map(str::to_owned);
1263
1264        let mut jobs = Vec::new();
1265        for &i in &todo {
1266            let (index, label, worktree) = {
1267                let c = &self.state.candidates[i];
1268                (c.index, c.label, c.worktree.clone())
1269            };
1270            let spec = self.roles.implementers[index].clone();
1271            let seat_key = format!("impl-{label}");
1272            let seat = self.seat(&seat_key, &spec.id);
1273            let instruction = self.state.instruction.clone();
1274            jobs.push(SeatJob {
1275                spec,
1276                seat,
1277                prompt: prompt::implement(
1278                    &instruction,
1279                    &worktree.to_string_lossy(),
1280                    &language,
1281                    brief.as_deref(),
1282                ),
1283                cwd: worktree,
1284                timeout,
1285                allow_write: true,
1286                sessions,
1287                artifacts: artifacts.clone(),
1288                stem: format!("impl-{label}"),
1289            });
1290        }
1291
1292        self.state.event(
1293            "implement",
1294            format!("{} candidates in parallel", jobs.len()),
1295        );
1296        // Kept so a seat whose CLI hung up can be asked again from the same
1297        // job: `wave` consumes what it is given. Mutable so `resume_quota_losses`
1298        // can update a seat's own entry once a fallback agent takes it over —
1299        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
1300        // whichever agent actually answered, not the one that quota'd out.
1301        let mut sent = jobs.clone();
1302        let cache = self.state.config.cache_dir();
1303        let ctx = WaveCtx {
1304            run: &run_id,
1305            node: "implement",
1306            prompts: &prompts,
1307            cache: cache.as_deref(),
1308            round: None,
1309        };
1310        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1311        self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1312            .await;
1313        self.resume_quota_losses(&mut results, &mut sent, &prompts, &run_id)
1314            .await;
1315        self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1316            .await;
1317
1318        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1319            let seat_key = seat.key.clone();
1320            // A quota fallback (`resume_quota_losses`) may have handed this
1321            // seat to a different agent than the one `prep` recorded on the
1322            // candidate; the stats tables and any later fixer-defaults-to-
1323            // winner's-author lookup must credit whoever actually answered —
1324            // unless every fallback also quota'd out, in which case nobody
1325            // actually answered and crediting the last agent tried would
1326            // erase every earlier agent's own quota loss from the stats
1327            // tables instead of just this one seat's.
1328            let agent = seat.agent.clone();
1329            let exhausted_the_fallback_chain = matches!(&out, AgentOutcome::Quota(_));
1330            self.state.seats.insert(seat.key.clone(), seat);
1331            let label = self.state.candidates[i].label;
1332            let worktree = self.state.candidates[i].worktree.clone();
1333            let base = self.state.base_commit.clone();
1334
1335            let (summary, duration, failed, verified_claim) = match out {
1336                AgentOutcome::Ok(o) => {
1337                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1338                    let failed = (!o.usable()).then(|| {
1339                        if o.timed_out {
1340                            "agent timed out".to_owned()
1341                        } else {
1342                            format!("agent exited with {:?}", o.exit_code)
1343                        }
1344                    });
1345                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
1346                    (text, o.duration_ms, failed, verified_claim)
1347                }
1348                // Left un-resumed by `resume_undelivered` (a dirty tree
1349                // already rescues the work, or there was no session left to
1350                // resume into) — reported like the ordinary failure it is,
1351                // never as if `o.text` (the CLI's raw error JSON) were an
1352                // answer.
1353                AgentOutcome::Dropped(o) => {
1354                    let why = o
1355                        .dropped
1356                        .as_ref()
1357                        .map(|d| d.why.as_str())
1358                        .unwrap_or("the CLI ended the stream without delivering its answer");
1359                    (
1360                        String::new(),
1361                        o.duration_ms,
1362                        Some(format!("the CLI dropped the stream ({why})")),
1363                        None,
1364                    )
1365                }
1366                AgentOutcome::Quota(o) => {
1367                    self.state.quota.push(QuotaLoss {
1368                        seat: seat_key,
1369                        node: "implement".to_owned(),
1370                        at: Timestamp::now(),
1371                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1372                    });
1373                    (
1374                        String::new(),
1375                        o.duration_ms,
1376                        Some("rate limited (quota); produced no change".to_owned()),
1377                        None,
1378                    )
1379                }
1380                AgentOutcome::Failed(e) => (String::new(), 0, Some(e), None),
1381            };
1382
1383            // Rescue anything the agent edited but never committed: an
1384            // uncommitted candidate would silently be an empty one.
1385            let rescued = match git::rescue_commit(
1386                &worktree,
1387                &format!("magi: candidate {label} (uncommitted work)"),
1388            )
1389            .await
1390            {
1391                Ok(r) => {
1392                    self.state.note_withheld("implement", &r.withheld);
1393                    r.committed
1394                }
1395                Err(_) => false,
1396            };
1397            let commits = git::commits_ahead(&worktree, &base, "HEAD")
1398                .await
1399                .unwrap_or(0);
1400            let patch = git::diff(&worktree, &base, "HEAD")
1401                .await
1402                .unwrap_or_default();
1403            let stat = git::diff_stat(&worktree, &base, "HEAD")
1404                .await
1405                .unwrap_or_default();
1406            let files = git::changed_files(&worktree, &base, "HEAD")
1407                .await
1408                .map(|f| f.len())
1409                .unwrap_or(0);
1410            write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1411
1412            let c = &mut self.state.candidates[i];
1413            if !exhausted_the_fallback_chain {
1414                c.agent = agent;
1415            }
1416            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1417            c.stat = stat;
1418            c.files = files;
1419            c.commits = commits;
1420            c.duration_ms = duration;
1421            c.empty = commits == 0 || patch.trim().is_empty();
1422            // An agent that failed but still produced a committed change stays
1423            // in the running: the patch is what gets judged, not the exit code.
1424            c.failed = match failed {
1425                Some(_) if c.empty => failed,
1426                _ => None,
1427            };
1428            // Only an empty candidate can be a verified no-op: a claim next
1429            // to a real patch is not what the marker is for, and `c.failed`
1430            // being `Some` here already implies `verified_claim` was never
1431            // set (see the guard above the match that produced it).
1432            c.verified_noop = if c.empty { verified_claim } else { None };
1433            let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
1434                (Some(e), _, _, _) => format!("candidate {label}: {e}"),
1435                (None, true, Some(_), _) => {
1436                    format!("candidate {label}: no change produced (agent-verified no-op)")
1437                }
1438                (None, true, None, _) => format!("candidate {label}: no change produced"),
1439                (None, false, _, true) => {
1440                    format!(
1441                        "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1442                    )
1443                }
1444                (None, false, _, false) => {
1445                    format!("candidate {label}: {files} files, {commits} commits")
1446                }
1447            };
1448            self.state.event("implement", note);
1449            self.state.save()?;
1450        }
1451
1452        self.after_implement()
1453    }
1454
1455    /// Ask again, once, for work a CLI did and then failed to hand over.
1456    ///
1457    /// [`agent::dropped_stream`] recognises the one shape observed: an error
1458    /// status with an empty response and a usage report showing output tokens,
1459    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
1460    /// seven minutes and 14,267 output tokens that arrived as an empty
1461    /// candidate, because `agy`'s own subscriber fell behind and hung up.
1462    ///
1463    /// Two conditions, and both matter:
1464    ///
1465    /// - **Only when the tree is untouched.** Often the agent has already
1466    ///   written its files and only the closing message was lost; the rescue
1467    ///   commit below picks that up and there is nothing to ask for. Re-asking
1468    ///   then would pay for a second implementation of work already on disk.
1469    /// - **Once.** A CLI that drops one stream can drop the next, and this
1470    ///   node is the most expensive in the graph.
1471    ///
1472    /// The re-ask is a resume, not a re-run: `has_context` is true because the
1473    /// dropped reply still carried its `conversation_id`, so the seat is asked
1474    /// to finish what it was doing rather than sent the whole task again. It
1475    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
1476    /// node's - for the same reason a re-ranked judge does: restating finished
1477    /// work is not the work.
1478    ///
1479    /// Unlike a quota this is worth retrying at all: a rate limit fails the
1480    /// same way until it resets, while an abandoned conversation is still
1481    /// there to be picked up.
1482    async fn resume_undelivered(
1483        &mut self,
1484        results: &mut [(usize, SeatState, AgentOutcome)],
1485        sent: &[SeatJob],
1486        prompts: &Prompts,
1487        run_id: &str,
1488    ) {
1489        for (wi, seat, out) in results.iter_mut() {
1490            let Some(dropped) = (match &*out {
1491                AgentOutcome::Dropped(o) => o.dropped.clone(),
1492                _ => None,
1493            }) else {
1494                continue;
1495            };
1496            let Some(job) = sent.get(*wi) else { continue };
1497            // Already on disk? Then only the closing message was lost.
1498            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1499                self.state.event(
1500                    "implement",
1501                    format!(
1502                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1503                         work is in the tree",
1504                        seat.key, dropped.output_tokens, dropped.why
1505                    ),
1506                );
1507                continue;
1508            }
1509            // The re-ask only makes sense as a resume: `resume_after_drop`
1510            // says nothing about the task, trusting the seat to still hold it.
1511            // Without a session to resume — sessions disabled, or this CLI's
1512            // drop shape happened not to carry a session id — that prompt
1513            // would open a brand-new conversation with no context at all,
1514            // which is worse than leaving this as the ordinary failure it
1515            // already is.
1516            if !has_context(&job.spec, seat, job.sessions) {
1517                self.state.event(
1518                    "implement",
1519                    format!(
1520                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1521                         is no session left to resume",
1522                        seat.key, dropped.output_tokens, dropped.why
1523                    ),
1524                );
1525                continue;
1526            }
1527            self.state.event(
1528                "implement",
1529                format!(
1530                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1531                     conversation",
1532                    seat.key, dropped.output_tokens, dropped.why
1533                ),
1534            );
1535            let mut retry = job.clone();
1536            retry.seat = seat.clone();
1537            retry.prompt = prompt::resume_after_drop(&dropped.why);
1538            retry.timeout = retry_budget(job.timeout, true);
1539            retry.stem = format!("{}-resume", job.stem);
1540            let cache = self.state.config.cache_dir();
1541            let ctx = WaveCtx {
1542                run: run_id,
1543                node: "implement",
1544                prompts,
1545                cache: cache.as_deref(),
1546                round: None,
1547            };
1548            let (resumed_seat, resumed) =
1549                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1550            *seat = resumed_seat;
1551            *out = resumed;
1552        }
1553    }
1554
1555    /// Fall an implement seat through to the next untried agent in the
1556    /// implementer roster when it lost to quota, instead of leaving the
1557    /// seat's loss final the moment one agent's account runs dry.
1558    ///
1559    /// Solo runs (`graph.candidates = 1`, `daemon::apply_solo`'s forced shape)
1560    /// are the motivating case: `Config::resolve_roles`'s `implementers`
1561    /// truncates to the single slot rotation picked, so a solo task whose one
1562    /// implementer hits quota mid-run used to have nothing else to try. This
1563    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
1564    /// unrotated roster — which is the only place the *other* candidates in
1565    /// the machine's roster still exist once `implementers` has been cut down
1566    /// to size.
1567    ///
1568    /// Walks forward from just past the seat's own original position in the
1569    /// roster, never wrapping back to the front: a later candidate slot (say
1570    /// `beta`, the roster's second entry) must fall through to the *next*
1571    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
1572    /// almost certainly a different candidate's own agent already — and once
1573    /// the roster's tail is exhausted there is nothing left to fall through
1574    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
1575    /// whole [`AgentSpec`]: a roster with the same id named twice must not
1576    /// let this retry that id forever. The loop keeps falling through until
1577    /// an attempt lands something other than `Quota` or the roster's tail
1578    /// runs out of untried ids, at which point the seat is left exactly as
1579    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
1580    /// `QuotaLoss` recorded, the candidate failed/empty.
1581    ///
1582    /// `sent` is taken mutably and updated with the fallback agent's spec:
1583    /// `resume_unconfirmed_commands`, which runs after this and also reads
1584    /// `sent`, must see whichever agent actually ended up answering the seat
1585    /// — reading the stale, original spec there would check session
1586    /// eligibility against the wrong CLI and could hand a fallback agent's
1587    /// session id to the agent that just lost the seat to quota.
1588    ///
1589    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
1590    /// — `self.seat` only reuses state when the agent id is unchanged, so
1591    /// handing it a different id already gets this for free. Reusing the old
1592    /// seat would resume a different CLI's session as if it were a
1593    /// continuation of this one.
1594    ///
1595    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
1596    /// a quota loss cuts an agent off mid-turn, so anything already in the
1597    /// tree is unfinished work, not a completed candidate a re-ask would pay
1598    /// for twice. A dirty tree is rescued into a commit first (the same
1599    /// neutral-identity rescue `implement`'s own outcome loop gives every
1600    /// candidate) so the next agent starts clean.
1601    ///
1602    /// The new agent gets the implementer's full prompt and full
1603    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
1604    /// it has no session and no context, and is implementing the task from
1605    /// nothing, unlike a resumed drop which is only restating work already
1606    /// done.
1607    ///
1608    /// Every intermediate `Quota` this loop absorbs is folded into a plain
1609    /// `implement` event, never into `self.state.quota` — that is what
1610    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
1611    /// go unspent, and a seat that ultimately recovered on its second or
1612    /// third agent is not the stalled panel that check exists to catch. Only
1613    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
1614    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
1615    /// outcome loop already has — this helper never pushes to it itself.
1616    async fn resume_quota_losses(
1617        &mut self,
1618        results: &mut [(usize, SeatState, AgentOutcome)],
1619        sent: &mut [SeatJob],
1620        prompts: &Prompts,
1621        run_id: &str,
1622    ) {
1623        let instruction = self.state.instruction.clone();
1624        let language = self.state.config.graph.language.clone();
1625        let brief = self
1626            .state
1627            .advice
1628            .as_ref()
1629            .and_then(|a| a.synthesis.as_deref())
1630            .map(str::to_owned);
1631        for (wi, seat, out) in results.iter_mut() {
1632            let Some(job) = sent.get_mut(*wi) else {
1633                continue;
1634            };
1635            // Where the seat's own original agent sits in the roster — the
1636            // fallback walk starts just past here, never at the front, so a
1637            // later candidate slot's quota loss does not fall back onto an
1638            // earlier slot's own agent.
1639            let start = self
1640                .roles
1641                .implementer_roster
1642                .iter()
1643                .position(|s| s.id == job.spec.id)
1644                .unwrap_or(0);
1645            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
1646            let mut fallback_attempt = 0usize;
1647            while matches!(&*out, AgentOutcome::Quota(_)) {
1648                let Some(next) =
1649                    next_untried_implementer(&self.roles.implementer_roster, start, &tried)
1650                        .cloned()
1651                else {
1652                    break;
1653                };
1654                tried.insert(next.id.clone());
1655                fallback_attempt += 1;
1656
1657                if let Ok(r) = git::rescue_commit(
1658                    &job.cwd,
1659                    &format!(
1660                        "magi: candidate {} (uncommitted work before quota fallback)",
1661                        seat.key
1662                    ),
1663                )
1664                .await
1665                {
1666                    self.state.note_withheld("implement", &r.withheld);
1667                }
1668
1669                self.state.event(
1670                    "implement",
1671                    format!(
1672                        "{}: rate limited (quota) on {}; retrying with {}",
1673                        seat.key, seat.agent, next.id
1674                    ),
1675                );
1676
1677                let new_seat = self.seat(&seat.key, &next.id);
1678                // Kept in sync on `sent` itself, not just the local retry: a
1679                // later helper (`resume_unconfirmed_commands`) reads `sent`
1680                // after this one returns and must see whichever agent is now
1681                // occupying the seat, not the one that just quota'd out —
1682                // otherwise it would judge session/continuation eligibility
1683                // by the wrong CLI and could resend a fallback's session id
1684                // to the agent that lost it the seat in the first place.
1685                job.spec = next.clone();
1686                let mut retry = job.clone();
1687                retry.seat = new_seat;
1688                retry.prompt = prompt::implement(
1689                    &instruction,
1690                    &job.cwd.to_string_lossy(),
1691                    &language,
1692                    brief.as_deref(),
1693                );
1694                retry.stem = format!("{}-quota-{}", job.stem, next.id);
1695                let cache = self.state.config.cache_dir();
1696                let ctx = WaveCtx {
1697                    run: run_id,
1698                    node: "implement",
1699                    prompts,
1700                    cache: cache.as_deref(),
1701                    round: None,
1702                };
1703                let (fallback_seat, fallback_out) = run_one(
1704                    retry,
1705                    Arc::clone(&self.sem),
1706                    &ctx,
1707                    &mut self.state,
1708                    fallback_attempt,
1709                )
1710                .await;
1711                *seat = fallback_seat;
1712                *out = fallback_out;
1713            }
1714        }
1715    }
1716
1717    /// Ask an implement seat's own CLI to confirm what it started, once, when
1718    /// its reply reported a command whose completion status it never
1719    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
1720    /// that does and does not mean.
1721    ///
1722    /// The completion contract this task asks for, extended to `implement`
1723    /// with the same signal `continue_fix_report` reads for the fixer,
1724    /// rather than a keyword search over the reply or a hard requirement on
1725    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
1726    /// a candidate's CLI turn ended cleanly while a test run it had started
1727    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
1728    /// named in it at all is untouched by this: `commands` is empty, so
1729    /// there is nothing to be unconfirmed.
1730    ///
1731    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
1732    /// this is not about recovering edits that might already be on disk, it
1733    /// is about a result the seat itself never vouched for, which resuming
1734    /// asks for regardless of what the tree already holds. Bounded to one
1735    /// attempt for the same reason `resume_undelivered` is — this is the
1736    /// most expensive node in the graph — and a seat that still cannot
1737    /// confirm on that attempt is left as whatever its (possibly still
1738    /// unconfirmed) reply says; this does not invent a new "failed" reason
1739    /// for a candidate that otherwise produced a real, committed change.
1740    async fn resume_unconfirmed_commands(
1741        &mut self,
1742        results: &mut [(usize, SeatState, AgentOutcome)],
1743        sent: &[SeatJob],
1744        prompts: &Prompts,
1745        run_id: &str,
1746    ) {
1747        for (wi, seat, out) in results.iter_mut() {
1748            let AgentOutcome::Ok(o) = &*out else {
1749                continue;
1750            };
1751            if !has_unconfirmed_command(&o.commands) {
1752                continue;
1753            }
1754            let Some(job) = sent.get(*wi) else { continue };
1755            if !has_context(&job.spec, seat, job.sessions) {
1756                self.state.event(
1757                    "implement",
1758                    format!(
1759                        "{}: the reply named a command whose own CLI never confirmed the exit \
1760                         status of, but there is no session left to resume",
1761                        seat.key
1762                    ),
1763                );
1764                continue;
1765            }
1766            self.state.event(
1767                "implement",
1768                format!(
1769                    "{}: the reply named a command whose own CLI never confirmed the exit \
1770                     status of; resuming the conversation",
1771                    seat.key
1772                ),
1773            );
1774            let mut retry = job.clone();
1775            retry.seat = seat.clone();
1776            retry.prompt = prompt::resume_incomplete(
1777                "a command in your last reply had no confirmed exit status",
1778            );
1779            retry.timeout = retry_budget(job.timeout, true);
1780            retry.stem = format!("{}-confirm", job.stem);
1781            let cache = self.state.config.cache_dir();
1782            let ctx = WaveCtx {
1783                run: run_id,
1784                node: "implement",
1785                prompts,
1786                cache: cache.as_deref(),
1787                round: None,
1788            };
1789            let (resumed_seat, resumed) =
1790                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1791            *seat = resumed_seat;
1792            *out = resumed;
1793        }
1794    }
1795
1796    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
1797    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
1798    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
1799    /// that motivated this.
1800    ///
1801    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
1802    /// own nudge loop already covers for judge/review/vote seats, and not a
1803    /// dropped stream, which [`Runner::resume_undelivered`] covers for
1804    /// implement seats: here the CLI turn genuinely finished while the node's
1805    /// own work — the fixer's account of what it did — had not. Gated purely
1806    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
1807    /// reply, never on any wording in it, so a fixer whose valid, first-try
1808    /// `FixReport` happens to mention having waited on a background test is
1809    /// never resumed — the `Ok(report)` branch at the call site returns
1810    /// before this is ever invoked.
1811    ///
1812    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
1813    /// attempt ([`retry_budget`]), nothing attempted once the session is
1814    /// gone, and a quota hit ends the loop immediately rather than retrying a
1815    /// rate limit that fails the same way again.
1816    async fn continue_fix_report(
1817        &mut self,
1818        mut seat: SeatState,
1819        parse_err: String,
1820        job: &SeatJob,
1821        prompts: &Prompts,
1822        run_id: &str,
1823        round: usize,
1824    ) -> (
1825        SeatState,
1826        Option<FixReport>,
1827        Option<String>,
1828        ContinuationRecord,
1829    ) {
1830        let mut last_err = parse_err;
1831        let mut cumulative_wait_ms = 0u64;
1832        let mut attempts = 0usize;
1833        loop {
1834            if !has_context(&job.spec, &seat, job.sessions) {
1835                self.state.event(
1836                    "fix",
1837                    format!(
1838                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
1839                         session left to resume into"
1840                    ),
1841                );
1842                let outcome = if attempts == 0 {
1843                    ContinuationOutcome::NoSession
1844                } else {
1845                    ContinuationOutcome::Exhausted
1846                };
1847                return (
1848                    seat,
1849                    None,
1850                    Some(format!("unparsable fix report: {last_err}")),
1851                    ContinuationRecord {
1852                        attempts,
1853                        cumulative_wait_ms,
1854                        outcome,
1855                    },
1856                );
1857            }
1858            if attempts >= MAX_FIX_CONTINUATIONS {
1859                self.state.event(
1860                    "fix",
1861                    format!(
1862                        "round {round}: fixer's reply still had no adoption report after \
1863                         {attempts} continuation(s) ({last_err}); giving up"
1864                    ),
1865                );
1866                return (
1867                    seat,
1868                    None,
1869                    Some(format!(
1870                        "unparsable fix report after {attempts} continuation(s): {last_err}"
1871                    )),
1872                    ContinuationRecord {
1873                        attempts,
1874                        cumulative_wait_ms,
1875                        outcome: ContinuationOutcome::Exhausted,
1876                    },
1877                );
1878            }
1879            attempts += 1;
1880            self.state.event(
1881                "fix",
1882                format!(
1883                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
1884                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
1885                ),
1886            );
1887            let mut retry = job.clone();
1888            retry.seat = seat.clone();
1889            retry.prompt = prompt::resume_incomplete(&last_err);
1890            retry.timeout = retry_budget(job.timeout, true);
1891            retry.stem = format!("{}-continue{attempts}", job.stem);
1892            let cache = self.state.config.cache_dir();
1893            let ctx = WaveCtx {
1894                run: run_id,
1895                node: "fix",
1896                prompts,
1897                cache: cache.as_deref(),
1898                round: Some(round),
1899            };
1900            let (resumed_seat, resumed_out) = run_one(
1901                retry,
1902                Arc::clone(&self.sem),
1903                &ctx,
1904                &mut self.state,
1905                attempts,
1906            )
1907            .await;
1908            seat = resumed_seat;
1909            match resumed_out {
1910                AgentOutcome::Ok(o) => {
1911                    cumulative_wait_ms += o.duration_ms;
1912                    match verdict::extract_json::<FixReport>(&o.text) {
1913                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
1914                            self.state.event(
1915                                "fix",
1916                                format!(
1917                                    "round {round}: fixer's adoption report recovered after \
1918                                     {attempts} continuation(s)"
1919                                ),
1920                            );
1921                            return (
1922                                seat,
1923                                Some(report),
1924                                None,
1925                                ContinuationRecord {
1926                                    attempts,
1927                                    cumulative_wait_ms,
1928                                    outcome: ContinuationOutcome::Resumed,
1929                                },
1930                            );
1931                        }
1932                        // The report parsed, but this same reply's own
1933                        // CommandEvidence — the identical record `state.jobs`
1934                        // renders — names a command whose CLI never
1935                        // confirmed an exit status. Read together, that is
1936                        // not a resolved answer: keep nudging rather than
1937                        // accept a report standing next to a command the
1938                        // seat's own CLI cannot vouch for.
1939                        Ok(_) => {
1940                            last_err = "the reply parsed, but it reported a command whose own CLI \
1941                                 never confirmed an exit status"
1942                                .to_owned();
1943                        }
1944                        Err(e) => last_err = e.to_string(),
1945                    }
1946                }
1947                AgentOutcome::Quota(o) => {
1948                    cumulative_wait_ms += o.duration_ms;
1949                    self.state.quota.push(QuotaLoss {
1950                        seat: seat.key.clone(),
1951                        node: "fix".to_owned(),
1952                        at: Timestamp::now(),
1953                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1954                    });
1955                    self.state.event(
1956                        "fix",
1957                        format!(
1958                            "round {round}: continuation rate limited (quota); not retrying now"
1959                        ),
1960                    );
1961                    return (
1962                        seat,
1963                        None,
1964                        Some("rate limited (quota) while recovering the fix report".to_owned()),
1965                        ContinuationRecord {
1966                            attempts,
1967                            cumulative_wait_ms,
1968                            outcome: ContinuationOutcome::QuotaLost,
1969                        },
1970                    );
1971                }
1972                AgentOutcome::Dropped(o) => {
1973                    cumulative_wait_ms += o.duration_ms;
1974                    let why = o
1975                        .dropped
1976                        .as_ref()
1977                        .map(|d| d.why.as_str())
1978                        .unwrap_or("the CLI ended the stream without delivering its answer");
1979                    last_err = format!("the CLI dropped the stream ({why})");
1980                }
1981                AgentOutcome::Failed(e) => last_err = e,
1982            }
1983        }
1984    }
1985
1986    fn after_implement(&mut self) -> Result<()> {
1987        // Scan every candidate patch once the set is complete.
1988        if self.state.leaks.is_empty() {
1989            let cfg = self.state.config.blind.clone();
1990            let mut leaks = Vec::new();
1991            for c in &self.state.candidates {
1992                let Some(patch) =
1993                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
1994                else {
1995                    continue;
1996                };
1997                leaks.extend(blind::scan(
1998                    &format!("candidate {} patch", c.label),
1999                    &patch,
2000                    &cfg.vendor_tokens,
2001                ));
2002            }
2003            if !leaks.is_empty() {
2004                let summary = leaks
2005                    .iter()
2006                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
2007                    .collect::<Vec<_>>()
2008                    .join(", ");
2009                match cfg.on_leak {
2010                    LeakPolicy::Fail => {
2011                        self.state.status = RunStatus::Failed;
2012                        self.state
2013                            .event("blind", format!("vendor text in a patch: {summary}"));
2014                        self.state.leaks = leaks;
2015                        self.state.save()?;
2016                        self.settle_questions();
2017                        bail!(
2018                            "blind.on_leak = \"fail\" and vendor text reached a \
2019                             judged patch: {summary}"
2020                        );
2021                    }
2022                    LeakPolicy::Redact => self.state.event(
2023                        "blind",
2024                        format!("redacting vendor text for judging: {summary}"),
2025                    ),
2026                    LeakPolicy::Warn => self.state.event(
2027                        "blind",
2028                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
2029                    ),
2030                }
2031                self.state.leaks = leaks;
2032            }
2033        }
2034
2035        if self.state.viable().is_empty() {
2036            if self.state.all_candidates_verified_noop() {
2037                // Every candidate agreed, with evidence the adoption guard
2038                // accepted, that nothing belongs in this worktree. That is
2039                // not the same fact as a candidate that simply failed to
2040                // write anything, and settling it as an ordinary `Failed`
2041                // (see `SCHEMA`'s doc for schema 10) is what let two of
2042                // task 391f's attempts burn a retry each re-discovering the
2043                // same already-landed fix. Terminal either way, so `judge`
2044                // must never run over an empty candidate set — unlike the
2045                // `Failed` branch below this returns `Ok`, not an error:
2046                // nothing here failed.
2047                self.state.status = RunStatus::VerifiedNoop;
2048                self.state.save()?;
2049                self.settle_questions();
2050                return Ok(());
2051            }
2052            self.state.status = RunStatus::Failed;
2053            self.state.save()?;
2054            self.settle_questions();
2055            bail!("no candidate produced a change; nothing to judge");
2056        }
2057        self.state.status = RunStatus::Judging;
2058        self.state.save()?;
2059        Ok(())
2060    }
2061
2062    // --------------------------------------------------------------- judge
2063
2064    async fn judge(&mut self) -> Result<()> {
2065        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2066        // agent files with `magi task add` name the run that paid for it. The
2067        // prompt overlay is cloned alongside it because the waves borrow it
2068        // while `self` is mutably borrowed by the node's own bookkeeping.
2069        let run_id = self.state.id.clone();
2070        let prompts = self.state.config.prompts.clone();
2071        if !self.state.judgements.is_empty() || self.state.judge_skipped {
2072            return Ok(());
2073        }
2074        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2075        if viable.len() == 1 {
2076            // Recorded so this is a one-time event: `judgements` stays empty
2077            // either way, which without this flag is indistinguishable from
2078            // "not yet judged" on the next reentry — and status is left
2079            // untouched, so a later node's conclusion (e.g. `Blocked` after
2080            // the review budget ran out) survives a resume instead of being
2081            // clobbered back to `Judging` by this node running again.
2082            self.state.judge_skipped = true;
2083            self.state.event(
2084                "judge",
2085                format!(
2086                    "only candidate {} produced a change; judging skipped",
2087                    viable[0].label
2088                ),
2089            );
2090            self.state.save()?;
2091            return Ok(());
2092        }
2093        self.state.status = RunStatus::Judging;
2094
2095        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2096        let language = self.state.config.graph.language.clone();
2097        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2098        let sessions = self.state.config.graph.sessions;
2099        let artifacts = agent::artifacts_dir(&self.state.dir());
2100        let root = self.state.worktree_root();
2101        let base_short = short(&self.state.base_commit);
2102
2103        let mut jobs = Vec::new();
2104        let mut orders = Vec::new();
2105        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2106            let order = blind::presentation_order(viable.len(), j, self.state.seed);
2107            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2108            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
2109            let seat_key = format!("judge-{}", j + 1);
2110            let seat = self.seat(&seat_key, &spec.id);
2111            jobs.push(SeatJob {
2112                prompt: prompt::judge(
2113                    &self.state.instruction,
2114                    &views,
2115                    self.roles.judges.len(),
2116                    &base_short,
2117                    &language,
2118                ),
2119                spec,
2120                seat,
2121                cwd: root.join(format!("judge-{}", j + 1)),
2122                timeout,
2123                allow_write: false,
2124                sessions,
2125                artifacts: artifacts.clone(),
2126                stem: format!("judge-{}", j + 1),
2127            });
2128        }
2129
2130        self.state.event(
2131            "judge",
2132            format!(
2133                "{} judges ranking {} candidates blind",
2134                jobs.len(),
2135                viable.len()
2136            ),
2137        );
2138        let labels_for_check = labels.clone();
2139        let mut quota_losses = Vec::new();
2140        let cache = self.state.config.cache_dir();
2141        let ctx = WaveCtx {
2142            run: &run_id,
2143            node: "judge",
2144            prompts: &prompts,
2145            cache: cache.as_deref(),
2146            round: None,
2147        };
2148        let results = ask_json_wave::<Ranking>(
2149            jobs,
2150            Arc::clone(&self.sem),
2151            self.state.config.graph.retries,
2152            &ctx,
2153            &mut quota_losses,
2154            &mut self.state,
2155            &move |r: &Ranking| r.validate(&labels_for_check),
2156        )
2157        .await;
2158        self.state.quota.extend(quota_losses);
2159
2160        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
2161            let agent_id = seat.agent.clone();
2162            self.state.seats.insert(seat.key.clone(), seat);
2163            let mut record = Judgement {
2164                judge: j + 1,
2165                seat: format!("judge-{}", j + 1),
2166                agent: agent_id,
2167                ranking: Vec::new(),
2168                reasons: BTreeMap::new(),
2169                confidence: None,
2170                order: orders[j].clone(),
2171                failed: None,
2172                duration_ms: 0,
2173            };
2174            match res {
2175                Ok((ranking, out)) => {
2176                    record.ranking = ranking.normalized();
2177                    record.reasons = ranking.reasons;
2178                    record.confidence = ranking.confidence;
2179                    record.duration_ms = out.duration_ms;
2180                    self.state.event(
2181                        "judge",
2182                        format!(
2183                            "judge {} ranked {}",
2184                            j + 1,
2185                            record.ranking.iter().collect::<String>()
2186                        ),
2187                    );
2188                }
2189                Err(e) => {
2190                    record.failed = Some(e.to_string());
2191                    self.state
2192                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
2193                }
2194            }
2195            self.state.judgements.push(record);
2196            self.state.save()?;
2197        }
2198        Ok(())
2199    }
2200
2201    // ---------------------------------------------------------- deliberate
2202
2203    async fn deliberate(&mut self) -> Result<()> {
2204        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2205        // agent files with `magi task add` name the run that paid for it. The
2206        // prompt overlay is cloned alongside it because the waves borrow it
2207        // while `self` is mutably borrowed by the node's own bookkeeping.
2208        let run_id = self.state.id.clone();
2209        let prompts = self.state.config.prompts.clone();
2210        if !self.state.deliberation.is_empty() {
2211            return Ok(());
2212        }
2213        let tops: Vec<char> = self
2214            .state
2215            .judgements
2216            .iter()
2217            .filter_map(|j| j.ranking.first().copied())
2218            .collect();
2219        let rounds = self.state.config.graph.deliberate_rounds;
2220        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
2221            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
2222                self.state.event(
2223                    "deliberate",
2224                    format!("judges agreed on {} outright; no deliberation", tops[0]),
2225                );
2226            }
2227            self.state.status = RunStatus::Voting;
2228            self.state.save()?;
2229            return Ok(());
2230        }
2231
2232        self.state.status = RunStatus::Deliberating;
2233        self.state.event(
2234            "deliberate",
2235            format!(
2236                "split: first choices were {} — opening {rounds} round(s)",
2237                tops.iter().collect::<String>()
2238            ),
2239        );
2240
2241        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2242        let language = self.state.config.graph.language.clone();
2243        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2244        let sessions = self.state.config.graph.sessions;
2245        let artifacts = agent::artifacts_dir(&self.state.dir());
2246        let root = self.state.worktree_root();
2247        let base_short = short(&self.state.base_commit);
2248
2249        // Judges argue in sequence so that a turn can answer the one before it;
2250        // that is the difference between deliberation and three parallel
2251        // monologues.
2252        for round in 1..=rounds {
2253            let mut turns: Vec<DeliberationTurn> = Vec::new();
2254            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2255                if self.state.judgements[j].failed.is_some() {
2256                    continue;
2257                }
2258                let seat_key = format!("judge-{}", j + 1);
2259                let mut seat = self.seat(&seat_key, &spec.id);
2260                let transcript = self.transcript(&turns, j);
2261                let context = if has_context(&spec, &seat, sessions) {
2262                    None
2263                } else {
2264                    Some(self.candidate_block(&viable, &base_short))
2265                };
2266                let text = prompt::deliberate(
2267                    &self.state.instruction,
2268                    context.as_deref(),
2269                    &transcript,
2270                    round,
2271                    rounds,
2272                    &language,
2273                );
2274                let job = SeatJob {
2275                    spec,
2276                    seat: seat.clone(),
2277                    prompt: text,
2278                    cwd: root.join(format!("judge-{}", j + 1)),
2279                    timeout,
2280                    allow_write: false,
2281                    sessions,
2282                    artifacts: artifacts.clone(),
2283                    stem: format!("delib-{round}-judge-{}", j + 1),
2284                };
2285                let cache = self.state.config.cache_dir();
2286                let ctx = WaveCtx {
2287                    run: &run_id,
2288                    node: "deliberate",
2289                    prompts: &prompts,
2290                    cache: cache.as_deref(),
2291                    round: None,
2292                };
2293                let (updated, out) =
2294                    run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2295                seat = updated;
2296                let agent_id = seat.agent.clone();
2297                let seat_key = seat.key.clone();
2298                self.state.seats.insert(seat.key.clone(), seat);
2299                let body = match out {
2300                    AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
2301                    // Never read the CLI's raw error JSON as this judge's
2302                    // position — skip the seat instead, the same as any other
2303                    // failed turn.
2304                    AgentOutcome::Dropped(o) => {
2305                        let why =
2306                            o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
2307                                "the CLI ended the stream without delivering its answer",
2308                            );
2309                        self.state.event(
2310                            "deliberate",
2311                            format!(
2312                                "judge {} skipped: the CLI dropped the stream ({why})",
2313                                j + 1
2314                            ),
2315                        );
2316                        continue;
2317                    }
2318                    AgentOutcome::Quota(o) => {
2319                        self.state.quota.push(QuotaLoss {
2320                            seat: seat_key,
2321                            node: "deliberate".to_owned(),
2322                            at: Timestamp::now(),
2323                            reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2324                        });
2325                        self.state.event(
2326                            "deliberate",
2327                            format!("judge {} skipped: rate limited (quota)", j + 1),
2328                        );
2329                        continue;
2330                    }
2331                    AgentOutcome::Failed(e) => {
2332                        self.state
2333                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
2334                        continue;
2335                    }
2336                };
2337                let tentative = verdict::extract_json::<Position>(&body)
2338                    .ok()
2339                    .and_then(|p| p.tentative)
2340                    .and_then(|s| s.trim().chars().next())
2341                    .map(|c| c.to_ascii_uppercase());
2342                self.state.event(
2343                    "deliberate",
2344                    format!(
2345                        "round {round}: judge {} now favours {}",
2346                        j + 1,
2347                        tentative.map_or("—".to_owned(), |c| c.to_string())
2348                    ),
2349                );
2350                turns.push(DeliberationTurn {
2351                    judge: j + 1,
2352                    agent: agent_id,
2353                    body: blind::sanitize_prose(&body, &self.state.config.blind),
2354                    tentative,
2355                });
2356            }
2357            self.state
2358                .deliberation
2359                .push(DeliberationRound { round, turns });
2360            self.state.save()?;
2361        }
2362
2363        self.state.status = RunStatus::Voting;
2364        self.state.save()?;
2365        Ok(())
2366    }
2367
2368    // ---------------------------------------------------------------- vote
2369
2370    async fn vote(&mut self) -> Result<()> {
2371        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2372        // agent files with `magi task add` name the run that paid for it. The
2373        // prompt overlay is cloned alongside it because the waves borrow it
2374        // while `self` is mutably borrowed by the node's own bookkeeping.
2375        let run_id = self.state.id.clone();
2376        let prompts = self.state.config.prompts.clone();
2377        if !self.state.votes.is_empty() {
2378            return Ok(());
2379        }
2380        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2381        if viable.len() == 1 {
2382            return Ok(());
2383        }
2384        self.state.status = RunStatus::Voting;
2385
2386        let language = self.state.config.graph.language.clone();
2387        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2388        let sessions = self.state.config.graph.sessions;
2389        let artifacts = agent::artifacts_dir(&self.state.dir());
2390        let root = self.state.worktree_root();
2391        let base_short = short(&self.state.base_commit);
2392        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2393
2394        let mut jobs = Vec::new();
2395        let mut seats_at = Vec::new();
2396        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2397            if self
2398                .state
2399                .judgements
2400                .get(j)
2401                .is_some_and(|r| r.failed.is_some())
2402            {
2403                continue;
2404            }
2405            let seat_key = format!("judge-{}", j + 1);
2406            let seat = self.seat(&seat_key, &spec.id);
2407            let mut text = prompt::final_vote(&viable, &language);
2408            if !has_context(&spec, &seat, sessions) {
2409                text = format!(
2410                    "{}\n\n# Candidates\n\n{}",
2411                    text,
2412                    self.candidate_block(&candidates, &base_short)
2413                );
2414            }
2415            jobs.push(SeatJob {
2416                spec,
2417                seat,
2418                prompt: text,
2419                cwd: root.join(format!("judge-{}", j + 1)),
2420                timeout,
2421                allow_write: false,
2422                sessions,
2423                artifacts: artifacts.clone(),
2424                stem: format!("vote-judge-{}", j + 1),
2425            });
2426            seats_at.push(j);
2427        }
2428
2429        self.state.event(
2430            "vote",
2431            format!(
2432                "collecting {} final votes one by one, privately",
2433                jobs.len()
2434            ),
2435        );
2436        let allowed = viable.clone();
2437        let mut quota_losses = Vec::new();
2438        let cache = self.state.config.cache_dir();
2439        let ctx = WaveCtx {
2440            run: &run_id,
2441            node: "vote",
2442            prompts: &prompts,
2443            cache: cache.as_deref(),
2444            round: None,
2445        };
2446        let results = ask_json_wave::<FinalVote>(
2447            jobs,
2448            Arc::clone(&self.sem),
2449            self.state.config.graph.retries,
2450            &ctx,
2451            &mut quota_losses,
2452            &mut self.state,
2453            &move |v: &FinalVote| match v.label() {
2454                Some(c) if allowed.contains(&c) => Ok(()),
2455                other => bail!("vote {other:?} is not one of {allowed:?}"),
2456            },
2457        )
2458        .await;
2459        self.state.quota.extend(quota_losses);
2460
2461        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
2462            let agent_id = seat.agent.clone();
2463            self.state.seats.insert(seat.key.clone(), seat);
2464            let initial = self
2465                .state
2466                .judgements
2467                .get(j)
2468                .and_then(|r| r.ranking.first().copied());
2469            let mut record = VoteRecord {
2470                judge: j + 1,
2471                agent: agent_id,
2472                vote: None,
2473                reason: String::new(),
2474                changed: false,
2475            };
2476            match res {
2477                Ok((v, _)) => {
2478                    record.vote = v.label();
2479                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2480                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
2481                    self.state.event(
2482                        "vote",
2483                        format!(
2484                            "judge {} voted {}{}",
2485                            j + 1,
2486                            record.vote.unwrap_or('?'),
2487                            if record.changed { " (changed)" } else { "" }
2488                        ),
2489                    );
2490                }
2491                Err(e) => {
2492                    self.state
2493                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
2494                }
2495            }
2496            self.state.votes.push(record);
2497            self.state.save()?;
2498        }
2499        Ok(())
2500    }
2501
2502    // --------------------------------------------------------------- tally
2503
2504    fn tally(&mut self) -> Result<()> {
2505        if self.state.tally.is_some() {
2506            return Ok(());
2507        }
2508        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2509        let tops: Vec<char> = self
2510            .state
2511            .judgements
2512            .iter()
2513            .filter_map(|j| j.ranking.first().copied())
2514            .collect();
2515        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
2516
2517        // A judge whose private vote failed still counted once, in the initial
2518        // ranking; using it beats discarding a whole seat.
2519        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2520        let mut cast: Vec<char> = Vec::new();
2521        for (i, j) in self.state.judgements.iter().enumerate() {
2522            let vote = self
2523                .state
2524                .votes
2525                .iter()
2526                .find(|v| v.judge == i + 1)
2527                .and_then(|v| v.vote)
2528                .or_else(|| j.ranking.first().copied());
2529            if let Some(v) = vote {
2530                *first_choice.entry(v).or_insert(0) += 1;
2531                cast.push(v);
2532            }
2533        }
2534
2535        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2536        for j in &self.state.judgements {
2537            let n = j.ranking.len();
2538            for (pos, label) in j.ranking.iter().enumerate() {
2539                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
2540            }
2541        }
2542
2543        let best = first_choice.values().copied().max().unwrap_or(0);
2544        let mut leaders: Vec<char> = first_choice
2545            .iter()
2546            .filter(|(_, v)| **v == best)
2547            .map(|(k, _)| *k)
2548            .collect();
2549        let mut tie_break = None;
2550        if leaders.len() > 1 {
2551            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
2552            let borda_leaders: Vec<char> = leaders
2553                .iter()
2554                .copied()
2555                .filter(|l| borda[l] == top_borda)
2556                .collect();
2557            tie_break = Some(if borda_leaders.len() == 1 {
2558                format!(
2559                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
2560                    leaders.len()
2561                )
2562            } else {
2563                format!(
2564                    "{} way tie on both first-choice votes and Borda points, broken by label order",
2565                    leaders.len()
2566                )
2567            });
2568            leaders = borda_leaders;
2569            leaders.sort_unstable();
2570        }
2571        let winner = *leaders
2572            .first()
2573            .or(viable.first())
2574            .context("no candidate to declare a winner from")?;
2575
2576        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
2577        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
2578        let deliberated = !self.state.deliberation.is_empty();
2579
2580        // Whose verdict is this? A rate-limited seat is absent even if it
2581        // ranked before the limit hit, so presence is measured against the
2582        // recorded losses, not just "did a ranking ever appear".
2583        let quota_seats: std::collections::BTreeSet<&str> =
2584            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2585        let mut present = 0usize;
2586        for (i, j) in self.state.judgements.iter().enumerate() {
2587            if quota_seats.contains(j.seat.as_str()) {
2588                continue;
2589            }
2590            let ranked = !j.ranking.is_empty() && j.failed.is_none();
2591            let voted = self
2592                .state
2593                .votes
2594                .iter()
2595                .any(|v| v.judge == i + 1 && v.vote.is_some());
2596            if ranked || voted {
2597                present += 1;
2598            }
2599        }
2600        // Strict majority of the configured panel. A bare majority is real
2601        // signal we can act on, while a minority verdict must never stand in
2602        // for a healthy one. A one-candidate run needs no panel at all, and
2603        // `judges` stays `0` rather than the roster size a panel that never
2604        // sat would otherwise be credited with.
2605        let needs_quorum = viable.len() > 1;
2606        let judges_total = if needs_quorum {
2607            self.roles.judges.len()
2608        } else {
2609            0
2610        };
2611        let quorum = if needs_quorum {
2612            judges_total / 2 + 1
2613        } else {
2614            0
2615        };
2616        let met_quorum = !needs_quorum || present >= quorum;
2617        let uncontested = (!needs_quorum).then(|| {
2618            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
2619        });
2620
2621        self.state.event(
2622            "tally",
2623            match &uncontested {
2624                Some(reason) => format!("winner {winner} — {reason}"),
2625                None => format!(
2626                    "winner {winner} — votes {} | initial {} | {} changed | \
2627                     {present}/{judges_total} judges{}",
2628                    first_choice
2629                        .iter()
2630                        .map(|(k, v)| format!("{k}:{v}"))
2631                        .collect::<Vec<_>>()
2632                        .join(" "),
2633                    if unanimous_initial {
2634                        "unanimous"
2635                    } else {
2636                        "split"
2637                    },
2638                    changed_votes,
2639                    if met_quorum {
2640                        String::new()
2641                    } else {
2642                        format!(" — below quorum ({quorum} required)")
2643                    },
2644                ),
2645            },
2646        );
2647        if !met_quorum {
2648            self.state.event(
2649                "stall",
2650                format!(
2651                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
2652                     the run stops here, resumable"
2653                ),
2654            );
2655        }
2656        self.state.tally = Some(Tally {
2657            first_choice,
2658            borda,
2659            winner,
2660            rankings: tops.len(),
2661            unanimous_initial,
2662            deliberated,
2663            changed_votes,
2664            unanimous_final,
2665            tie_break,
2666            judges: judges_total,
2667            present,
2668            quorum,
2669            met_quorum,
2670            uncontested,
2671        });
2672        self.state.status = if met_quorum {
2673            RunStatus::Reviewing
2674        } else {
2675            RunStatus::Stalled
2676        };
2677        self.state.save()?;
2678        Ok(())
2679    }
2680
2681    // ------------------------------------------------------------- recover
2682
2683    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
2684    /// resumed toward completion once the transient cause clears.
2685    ///
2686    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
2687    /// it toward the quorum, which is exactly the set of seats whose absence
2688    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
2689    /// not depend on which node happened to hit the limit), or an ordinary
2690    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
2691    /// seat is never disturbed.
2692    ///
2693    /// A seat that now answers with a usable ranking is "recovered": its
2694    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
2695    /// `tally` counts it present again), and its vote re-collected. A seat that
2696    /// still fails keeps its loss and stays absent.
2697    ///
2698    /// Returns `true` when the re-tally restores the quorum (the run may proceed
2699    /// to review/gate/merge), `false` when it is still below quorum (the run
2700    /// stays `Stalled`, still resumable for a later retry).
2701    #[allow(clippy::too_many_lines)]
2702    async fn recover_stall(&mut self) -> Result<bool> {
2703        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2704        // agent files with `magi task add` name the run that paid for it. The
2705        // prompt overlay is cloned alongside it because the waves borrow it
2706        // while `self` is mutably borrowed by the node's own bookkeeping.
2707        let run_id = self.state.id.clone();
2708        let prompts = self.state.config.prompts.clone();
2709        // Absent seats = quota-lost at any node, or failed outright. Mirroring
2710        // `tally`'s presence test (rather than the old quota-judge/vote filter)
2711        // is what keeps a non-quota collapse — or a quota loss recorded at the
2712        // deliberate node — from being a permanent dead-end on `--resume`.
2713        let quota_seats: BTreeSet<&str> =
2714            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2715        let absent: Vec<String> = self
2716            .state
2717            .judgements
2718            .iter()
2719            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
2720            .map(|j| j.seat.clone())
2721            .collect();
2722        if absent.is_empty() {
2723            return Ok(false);
2724        }
2725        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2726        if viable.len() <= 1 {
2727            return Ok(false);
2728        }
2729        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2730        let language = self.state.config.graph.language.clone();
2731        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2732        let sessions = self.state.config.graph.sessions;
2733        let artifacts = agent::artifacts_dir(&self.state.dir());
2734        let root = self.state.worktree_root();
2735        let base_short = short(&self.state.base_commit);
2736        let candidates: Vec<Candidate> = viable.clone();
2737
2738        // Map each absent seat key to its 0-based position in `roles.judges`.
2739        let mut positions: Vec<usize> = absent
2740            .iter()
2741            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
2742            .collect();
2743        if positions.is_empty() {
2744            return Ok(false);
2745        }
2746        positions.sort_unstable();
2747        positions.dedup();
2748
2749        // Re-rank the lost seats, one blind prompt each.
2750        let mut judge_jobs = Vec::new();
2751        for &j in &positions {
2752            let order = blind::presentation_order(viable.len(), j, self.state.seed);
2753            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2754            let seat_key = format!("judge-{}", j + 1);
2755            let spec = self.roles.judges[j].clone();
2756            let seat = self.seat(&seat_key, &spec.id);
2757            judge_jobs.push(SeatJob {
2758                spec,
2759                seat,
2760                prompt: prompt::judge(
2761                    &self.state.instruction,
2762                    &views,
2763                    self.roles.judges.len(),
2764                    &base_short,
2765                    &language,
2766                ),
2767                cwd: root.join(seat_key),
2768                timeout,
2769                allow_write: false,
2770                sessions,
2771                artifacts: artifacts.clone(),
2772                stem: format!("judge-{}-recover", j + 1),
2773            });
2774        }
2775
2776        let labels_for_check = labels.clone();
2777        let mut judge_losses = Vec::new();
2778        let retries = self.state.config.graph.retries;
2779        let cache = self.state.config.cache_dir();
2780        let ctx = WaveCtx {
2781            run: &run_id,
2782            node: "judge",
2783            prompts: &prompts,
2784            cache: cache.as_deref(),
2785            round: None,
2786        };
2787        let results = ask_json_wave::<Ranking>(
2788            judge_jobs,
2789            Arc::clone(&self.sem),
2790            retries,
2791            &ctx,
2792            &mut judge_losses,
2793            &mut self.state,
2794            &move |r: &Ranking| r.validate(&labels_for_check),
2795        )
2796        .await;
2797
2798        // Refresh the judgement of every seat that ranked again.
2799        let mut recovered: BTreeSet<usize> = BTreeSet::new();
2800        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
2801            self.state.seats.insert(seat.key.clone(), seat);
2802            let record = &mut self.state.judgements[j];
2803            match res {
2804                Ok((ranking, out)) => {
2805                    record.ranking = ranking.normalized();
2806                    record.reasons = ranking.reasons;
2807                    record.confidence = ranking.confidence;
2808                    record.failed = None;
2809                    record.duration_ms = out.duration_ms;
2810                    recovered.insert(j);
2811                    self.state.event(
2812                        "recover",
2813                        format!("judge {} ranked again after the limit", j + 1),
2814                    );
2815                }
2816                Err(e) => {
2817                    self.state
2818                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
2819                }
2820            }
2821        }
2822
2823        // Re-ask the votes of the seats that recovered a ranking.
2824        let mut vote_jobs = Vec::new();
2825        let mut vote_pos: Vec<usize> = Vec::new();
2826        for &j in &recovered {
2827            let seat_key = format!("judge-{}", j + 1);
2828            let spec = self.roles.judges[j].clone();
2829            let seat = self.seat(&seat_key, &spec.id);
2830            let mut text = prompt::final_vote(&labels, &language);
2831            if !has_context(&spec, &seat, sessions) {
2832                text = format!(
2833                    "{}\n\n# Candidates\n\n{}",
2834                    text,
2835                    self.candidate_block(&candidates, &base_short)
2836                );
2837            }
2838            vote_jobs.push(SeatJob {
2839                spec,
2840                seat,
2841                prompt: text,
2842                cwd: root.join(seat_key),
2843                timeout,
2844                allow_write: false,
2845                sessions,
2846                artifacts: artifacts.clone(),
2847                stem: format!("vote-judge-{}-recover", j + 1),
2848            });
2849            vote_pos.push(j);
2850        }
2851        let allowed = labels.clone();
2852        let mut vote_losses = Vec::new();
2853        let vote_retries = self.state.config.graph.retries;
2854        let vote_cache = self.state.config.cache_dir();
2855        let ctx = WaveCtx {
2856            run: &run_id,
2857            node: "vote",
2858            prompts: &prompts,
2859            cache: vote_cache.as_deref(),
2860            round: None,
2861        };
2862        let votes = ask_json_wave::<FinalVote>(
2863            vote_jobs,
2864            Arc::clone(&self.sem),
2865            vote_retries,
2866            &ctx,
2867            &mut vote_losses,
2868            &mut self.state,
2869            &move |v: &FinalVote| match v.label() {
2870                Some(c) if allowed.contains(&c) => Ok(()),
2871                other => bail!("vote {other:?} is not one of {allowed:?}"),
2872            },
2873        )
2874        .await;
2875        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
2876            let agent_id = seat.agent.clone();
2877            self.state.seats.insert(seat.key.clone(), seat);
2878            match res {
2879                Ok((v, _)) => {
2880                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
2881                        rec.vote = v.label();
2882                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2883                    } else {
2884                        self.state.votes.push(VoteRecord {
2885                            judge: j + 1,
2886                            agent: agent_id,
2887                            vote: v.label(),
2888                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
2889                            changed: false,
2890                        });
2891                    }
2892                    self.state.event(
2893                        "recover",
2894                        format!("judge {} voted again after the limit", j + 1),
2895                    );
2896                }
2897                Err(e) => {
2898                    self.state
2899                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
2900                }
2901            }
2902        }
2903
2904        // A seat that ranked again is present even if its re-vote failed —
2905        // `tally` falls back to the initial ranking's first choice — so clear
2906        // its quota loss. Seats that still fail keep theirs and stay absent.
2907        let recovered_keys: BTreeSet<String> = recovered
2908            .iter()
2909            .map(|&j| format!("judge-{}", j + 1))
2910            .collect();
2911        self.state
2912            .quota
2913            .retain(|q| !recovered_keys.contains(&q.seat));
2914        // A seat that hit the limit again is a fresh loss, not the old one:
2915        // replace the stale entry so the history stays one-per-seat and the
2916        // daemon can tell this attempt's loss from a previous session's.
2917        for loss in judge_losses.into_iter().chain(vote_losses) {
2918            if recovered_keys.contains(&loss.seat) {
2919                continue;
2920            }
2921            self.state.quota.retain(|q| q.seat != loss.seat);
2922            self.state.quota.push(loss);
2923        }
2924
2925        // Recompute the verdict from the refreshed panel.
2926        self.state.tally = None;
2927        self.tally()?;
2928        Ok(self
2929            .state
2930            .tally
2931            .as_ref()
2932            .map(|t| t.met_quorum)
2933            .unwrap_or(false))
2934    }
2935
2936    // ----------------------------------------------------------------- fold
2937
2938    async fn fold_losers(&mut self) -> Result<()> {
2939        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
2940            return Ok(());
2941        };
2942        let repo = self.state.repo.clone();
2943        let mut folded = Vec::new();
2944        for i in 0..self.state.candidates.len() {
2945            let c = &self.state.candidates[i];
2946            if c.label == winner || c.folded {
2947                continue;
2948            }
2949            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
2950            git::worktree_remove(&repo, &wt).await.ok();
2951            git::branch_delete(&repo, &branch).await.ok();
2952            self.state.candidates[i].folded = true;
2953            folded.push(label.to_string());
2954        }
2955        // The judges are finished; their checkouts are pure cost from here.
2956        let root = self.state.worktree_root();
2957        for j in 1..=self.roles.judges.len() {
2958            let wt = root.join(format!("judge-{j}"));
2959            if wt.exists() {
2960                git::worktree_remove(&repo, &wt).await.ok();
2961            }
2962        }
2963        // The design-deliberation stage is finished by the time a tally
2964        // exists — same reasoning as the judges above.
2965        if self.state.config.graph.advise {
2966            for k in 1..=self.state.config.graph.advisors {
2967                let wt = root.join(format!("advisor-{k}"));
2968                if wt.exists() {
2969                    git::worktree_remove(&repo, &wt).await.ok();
2970                }
2971            }
2972        }
2973        if !folded.is_empty() {
2974            self.state
2975                .event("fold", format!("folded candidates {}", folded.join(", ")));
2976            self.state.save()?;
2977        }
2978        Ok(())
2979    }
2980
2981    // ------------------------------------------------------------ base sync
2982
2983    /// Land the winner's tree on the current tip of `<remote>/<base>` before
2984    /// anything verifies it.
2985    ///
2986    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
2987    /// read whatever is checked out in the winner's worktree. Left alone that
2988    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
2989    /// it when the run *branched* - and a run takes long enough that the base
2990    /// has usually moved by the time it gets here. A gate that ran there
2991    /// answers "green on the commit this run started from", not "green on
2992    /// what is about to land", and the difference showed up three times in
2993    /// one day as a green run whose merge would have reverted a file another
2994    /// pull request had already landed.
2995    ///
2996    /// Reuses [`git::rebase_branch_in_temp`] rather than a second
2997    /// implementation of the same idea: `land::Step::Rebase` already worked
2998    /// out the rules - throwaway worktree, conflict stops and reports rather
2999    /// than feeding a fixer, nothing runs in the primary tree - and a second
3000    /// rebase path is exactly the kind of drift `resolve_base`'s own doc
3001    /// warns about ("two answers to a question nobody notices until a diff is
3002    /// wrong").
3003    ///
3004    /// Bounded by [`BASE_SYNC_ROUNDS`], counted in `state.base_sync.attempts`
3005    /// so it survives a park/resume. A conflict or a push failure sets
3006    /// `state.base_sync.conflict` and leaves the branch and worktree exactly
3007    /// as they were - untouched, for a person to look at - which is also what
3008    /// makes re-entering this function afterwards a no-op instead of a second
3009    /// attempt at the same wall.
3010    async fn sync_to_base(&mut self) -> Result<()> {
3011        if self
3012            .state
3013            .base_sync
3014            .as_ref()
3015            .is_some_and(|s| s.conflict.is_some())
3016        {
3017            return Ok(());
3018        }
3019        let Some(winner) = self.state.winner().cloned() else {
3020            return Ok(());
3021        };
3022
3023        let repo = self.state.repo.clone();
3024        let remote = self.state.config.merge.remote.clone();
3025        let base_branch = self.state.base_branch.clone();
3026        let tracking = format!("{remote}/{base_branch}");
3027
3028        git::fetch(&repo, &remote, &base_branch).await.ok();
3029        // No network, or the remote never had this branch: `resolve_base`
3030        // already treats that as non-fatal at branch time, and a run that got
3031        // this far must not be blocked by it here either.
3032        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
3033            return Ok(());
3034        };
3035
3036        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3037        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
3038        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
3039
3040        if behind == 0 {
3041            self.state.base_sync = Some(BaseSync {
3042                tip,
3043                behind: 0,
3044                attempts,
3045                conflict: None,
3046            });
3047            self.state.save()?;
3048            return Ok(());
3049        }
3050
3051        if attempts >= BASE_SYNC_ROUNDS {
3052            let why = format!(
3053                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
3054                 rebase(s); rebasing again would only race it",
3055                winner.branch
3056            );
3057            self.state.status = RunStatus::Blocked;
3058            self.state.base_sync = Some(BaseSync {
3059                tip,
3060                behind,
3061                attempts,
3062                conflict: Some(why.clone()),
3063            });
3064            self.state.event("land", why);
3065            self.state.save()?;
3066            return Ok(());
3067        }
3068
3069        self.state.event(
3070            "land",
3071            format!(
3072                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
3073                winner.branch
3074            ),
3075        );
3076        self.state.save()?;
3077
3078        let scratch = self.state.dir().join("base-sync");
3079        let rebased = git::rebase_branch_in_temp(&repo, &scratch, &winner.branch, &tracking).await;
3080        let attempts = attempts + 1;
3081        match rebased {
3082            Ok(None) => {
3083                // The branch ref moved, but a worktree that already had it
3084                // checked out (the winner's) was not told; sync its index and
3085                // files before anything reads them.
3086                git::sync_to_head(&winner.worktree).await?;
3087                self.state.base_sync = Some(BaseSync {
3088                    tip: tip.clone(),
3089                    behind: 0,
3090                    attempts,
3091                    conflict: None,
3092                });
3093                self.state
3094                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
3095            }
3096            Ok(Some(conflict)) => {
3097                let why = format!(
3098                    "{} conflicts with {tracking} and did not rebase: {}",
3099                    winner.branch,
3100                    conflict.chars().take(600).collect::<String>()
3101                );
3102                self.state.status = RunStatus::Blocked;
3103                self.state.base_sync = Some(BaseSync {
3104                    tip,
3105                    behind,
3106                    attempts,
3107                    conflict: Some(why.clone()),
3108                });
3109                self.state.event("land", why);
3110            }
3111            Err(e) => {
3112                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
3113                self.state.status = RunStatus::Blocked;
3114                self.state.base_sync = Some(BaseSync {
3115                    tip,
3116                    behind,
3117                    attempts,
3118                    conflict: Some(why.clone()),
3119                });
3120                self.state.event("land", why);
3121            }
3122        }
3123        self.state.save()?;
3124        Ok(())
3125    }
3126
3127    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
3128    /// last landed the winner on, once it has run, else the commit the run
3129    /// branched from.
3130    ///
3131    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
3132    /// and `vote` all happen before there is a winner to rebase, so they
3133    /// compare every candidate against the branch point on purpose, and a
3134    /// base that moves after they are already done cannot change an answer
3135    /// they already gave.
3136    fn landing_base(&self) -> String {
3137        self.state
3138            .base_sync
3139            .as_ref()
3140            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
3141    }
3142
3143    // ------------------------------------------------------- operator fix
3144
3145    /// Route specific, already-recorded review findings to a fixer for a
3146    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
3147    /// entry point.
3148    ///
3149    /// Distinct from `review_loop`'s own fix step in three ways: it never
3150    /// runs a reviewer wave, it never spends review-round budget, and what
3151    /// happened is recorded as an [`OperatorFixRequest`] appended to
3152    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
3153    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
3154    /// and vote must never be rewritten to look like a manufactured blocking
3155    /// verdict.
3156    ///
3157    /// Only meaningful once review has actually concluded: `Ready` (handed
3158    /// off with findings still open, or simply concluded clean while minor
3159    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
3160    /// gate failed). Everything else is refused: a run still in progress
3161    /// should simply be resumed, and a `Merged` run's branch has already
3162    /// landed — reopening *this* run's own record cannot change that, so the
3163    /// answer there is a fresh `magi review <branch>`.
3164    ///
3165    /// A real commit here re-verifies through a fresh, ordinary review-only
3166    /// run on the same branch ([`Self::review`]) rather than reopening this
3167    /// run's own `review_loop`: once any round in this run's history went
3168    /// clean, `review_conclusion` treats that as permanent by design (the
3169    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
3170    /// way to force one more genuine reviewer wave out of *this* run without
3171    /// either rewriting history or weakening that guarantee for every other
3172    /// caller. A review-only run costs nothing extra — no implementation, no
3173    /// judging, no vote — and exercises the exact same review → verify →
3174    /// gate → (human) merge path, unmodified.
3175    pub async fn fix_selected(
3176        &mut self,
3177        ids: &[String],
3178        reason: &str,
3179        allow_stale: bool,
3180    ) -> Result<()> {
3181        let reason = reason.trim();
3182        if reason.is_empty() {
3183            bail!("a fix request needs a reason — that is the operator's own record of why");
3184        }
3185        if ids.is_empty() {
3186            bail!("no finding id given");
3187        }
3188        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
3189            bail!(
3190                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
3191                 has already concluded — can be given a targeted fix. A run still \
3192                 in progress should simply be resumed; a `merged` run's branch has \
3193                 already landed, so its answer is a fresh `magi review <branch>`, \
3194                 not reopening this run's own record",
3195                self.state.id,
3196                self.state.status.as_str()
3197            );
3198        }
3199        let Some(winner) = self.state.winner().cloned() else {
3200            bail!("run {} has no winning candidate to fix", self.state.id);
3201        };
3202        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
3203            bail!(
3204                "branch `{}` no longer exists; this run cannot be extended",
3205                winner.branch
3206            );
3207        }
3208        let home = crate::run::home();
3209        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
3210            bail!(
3211                "run {} is currently being worked on by another magi process",
3212                self.state.id
3213            );
3214        }
3215        // Held for the rest of this call, including the follow-up review
3216        // below: two `magi fix` invocations against the same run must not
3217        // both reach the worktree manipulation further down, which would
3218        // otherwise race to remove and recreate the same directory — see
3219        // [`FixClaim`]'s own doc.
3220        let _claim = FixClaim::acquire(&self.state.dir())?;
3221
3222        // Resolve every id before spending anything — an unknown id refuses
3223        // the whole request rather than silently dropping it — and dedup
3224        // while keeping the operator's own order.
3225        let mut seen = BTreeSet::new();
3226        let mut findings = Vec::new();
3227        let mut missing = Vec::new();
3228        for id in ids {
3229            if !seen.insert(id.clone()) {
3230                continue;
3231            }
3232            match self.state.finding(id) {
3233                Some((round, rec, f)) => findings.push(OperatorFixFinding {
3234                    id: f.id.clone(),
3235                    severity: f.severity,
3236                    reviewer_vote: rec.vote,
3237                    round: round.round,
3238                    round_head: round.head.clone(),
3239                    reviewer: rec.reviewer,
3240                    agent: rec.agent.clone(),
3241                    file: f.file.clone(),
3242                    line: f.line,
3243                    title: f.title.clone(),
3244                    detail: f.detail.clone(),
3245                    outcome: OperatorFixOutcome::Pending,
3246                }),
3247                None => missing.push(id.clone()),
3248            }
3249        }
3250        if !missing.is_empty() {
3251            bail!(
3252                "unknown finding id(s): {}; nothing was changed",
3253                missing.join(", ")
3254            );
3255        }
3256
3257        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
3258        let stale_details: Vec<(String, String)> = findings
3259            .iter()
3260            .filter(|f| f.round_head != head_at_request)
3261            .map(|f| (f.id.clone(), f.round_head.clone()))
3262            .collect();
3263        let stale = !stale_details.is_empty();
3264        if stale && !allow_stale {
3265            bail!(
3266                "the branch has moved since some finding(s) were raised — {} — now \
3267                 at {}; pass --allow-stale to fix anyway, or re-run review first",
3268                stale_details
3269                    .iter()
3270                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
3271                    .collect::<Vec<_>>()
3272                    .join(", "),
3273                short(&head_at_request)
3274            );
3275        }
3276
3277        let request = OperatorFixRequest {
3278            requested_at: Timestamp::now(),
3279            reason: reason.to_owned(),
3280            findings,
3281            head_at_request: head_at_request.clone(),
3282            allow_stale,
3283            stale,
3284            fix: None,
3285            result_head: None,
3286            follow_up_review_run: None,
3287        };
3288        self.state.event(
3289            "fix",
3290            format!(
3291                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
3292                request.findings.len(),
3293                request
3294                    .findings
3295                    .iter()
3296                    .map(|f| f.id.as_str())
3297                    .collect::<Vec<_>>()
3298                    .join(", "),
3299            ),
3300        );
3301        // Recorded now, before any worktree work or the fixer call itself —
3302        // and re-saved at each checkpoint below: a crash at any point after
3303        // this (mid fixer call, mid follow-up review) must not lose the fact
3304        // that this was requested, for which findings, and why. Everything
3305        // past this point reads and writes through `request_index` rather
3306        // than a local variable, since `request` itself is moved here.
3307        self.state.operator_fixes.push(request);
3308        self.state.save()?;
3309        let request_index = self.state.operator_fixes.len() - 1;
3310
3311        // A fresh, dedicated worktree for this one call, never the winner's
3312        // own worktree in place: that one may already be gone (folded away),
3313        // and reusing it in place would leave the branch checked out there
3314        // when the follow-up review below tries to check it out again. Freed
3315        // immediately after, either way — but only once confirmed clean:
3316        // `worktree_remove` is a `git worktree remove --force`, which would
3317        // otherwise discard uncommitted work left there by the operator or
3318        // another process before this had a chance to even look at it.
3319        if winner.worktree.exists() {
3320            // Lockfiles a rescue commit withheld stay untracked on purpose and
3321            // are already recorded; they are not the operator's work to protect.
3322            let dirty = git::git(
3323                &winner.worktree,
3324                &["status", "--porcelain", "--untracked-files=all"],
3325            )
3326            .await?;
3327            let only_withheld = dirty.lines().all(|l| {
3328                l.strip_prefix("?? ")
3329                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
3330            });
3331            if !only_withheld {
3332                bail!(
3333                    "`{}` has uncommitted changes; refusing to touch it — commit or \
3334                     discard them first",
3335                    winner.worktree.display()
3336                );
3337            }
3338            git::worktree_remove(&self.state.repo, &winner.worktree)
3339                .await
3340                .ok();
3341        }
3342        let fix_worktree = self.state.worktree_root().join("operator-fix");
3343        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
3344        git::git(
3345            &self.state.repo,
3346            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
3347        )
3348        .await
3349        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
3350        if !git::is_clean(&fix_worktree).await? {
3351            git::worktree_remove(&self.state.repo, &fix_worktree)
3352                .await
3353                .ok();
3354            bail!(
3355                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
3356                winner.branch
3357            );
3358        }
3359
3360        let run_id = self.state.id.clone();
3361        let prompts = self.state.config.prompts.clone();
3362        let language = self.state.config.graph.language.clone();
3363        let sessions = self.state.config.graph.sessions;
3364        let artifacts = agent::artifacts_dir(&self.state.dir());
3365        let (fix_spec, fix_seat_key) = match &self.roles.fixer {
3366            Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3367            _ => (
3368                self.state
3369                    .config
3370                    .agent(&winner.agent)
3371                    .cloned()
3372                    .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3373                format!("impl-{}", winner.label),
3374            ),
3375        };
3376        let seat = self.seat(&fix_seat_key, &fix_spec.id);
3377        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
3378            .findings
3379            .iter()
3380            .map(|f| Finding {
3381                id: f.id.clone(),
3382                severity: f.severity,
3383                file: f.file.clone(),
3384                line: f.line,
3385                title: f.title.clone(),
3386                detail: f.detail.clone(),
3387            })
3388            .collect();
3389        let job = SeatJob {
3390            prompt: prompt::operator_fix(
3391                &self.state.instruction,
3392                &finding_list,
3393                reason,
3394                &stale_details,
3395                &head_at_request,
3396                &language,
3397            ),
3398            spec: fix_spec.clone(),
3399            seat,
3400            cwd: fix_worktree.clone(),
3401            timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
3402            allow_write: true,
3403            sessions,
3404            artifacts: artifacts.clone(),
3405            stem: "operator-fix".to_owned(),
3406        };
3407        let cache = self.state.config.cache_dir();
3408        let ctx = WaveCtx {
3409            run: &run_id,
3410            node: "fix",
3411            prompts: &prompts,
3412            cache: cache.as_deref(),
3413            round: None,
3414        };
3415        let (seat, out) =
3416            run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3417        let agent_id = seat.agent.clone();
3418
3419        let mut fix = FixRecord {
3420            agent: agent_id,
3421            addressed: Vec::new(),
3422            rejected: Vec::new(),
3423            notes: String::new(),
3424            committed: false,
3425            failed: None,
3426            duration_ms: 0,
3427            continuation: None,
3428        };
3429        let mut final_seat = seat.clone();
3430        match out {
3431            AgentOutcome::Ok(o) => {
3432                fix.duration_ms = o.duration_ms;
3433                let parsed = verdict::extract_json::<FixReport>(&o.text);
3434                let incomplete_reason = match &parsed {
3435                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
3436                        "the reply parsed, but it reported a command whose own CLI \
3437                         never confirmed an exit status"
3438                            .to_owned(),
3439                    ),
3440                    Ok(_) => None,
3441                    Err(e) => Some(e.to_string()),
3442                };
3443                match incomplete_reason {
3444                    None => {
3445                        let report = parsed.expect("checked Ok above");
3446                        fix.addressed = report.addressed;
3447                        fix.rejected = report.rejected;
3448                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
3449                    }
3450                    Some(reason) => {
3451                        let (resumed_seat, resolved, failure, cont) = self
3452                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
3453                            .await;
3454                        fix.duration_ms += cont.cumulative_wait_ms;
3455                        fix.continuation = Some(cont);
3456                        final_seat = resumed_seat;
3457                        match resolved {
3458                            Some(report) => {
3459                                fix.addressed = report.addressed;
3460                                fix.rejected = report.rejected;
3461                                fix.notes =
3462                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
3463                            }
3464                            None => fix.failed = failure,
3465                        }
3466                    }
3467                }
3468            }
3469            AgentOutcome::Dropped(o) => {
3470                fix.duration_ms = o.duration_ms;
3471                let why = o
3472                    .dropped
3473                    .as_ref()
3474                    .map(|d| d.why.as_str())
3475                    .unwrap_or("the CLI ended the stream without delivering its answer");
3476                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
3477            }
3478            AgentOutcome::Quota(o) => {
3479                self.state.quota.push(QuotaLoss {
3480                    seat: final_seat.key.clone(),
3481                    node: "fix".to_owned(),
3482                    at: Timestamp::now(),
3483                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3484                });
3485                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
3486            }
3487            AgentOutcome::Failed(e) => fix.failed = Some(e),
3488        }
3489        if fix.continuation.is_none() {
3490            fix.continuation = Some(ContinuationRecord::not_needed());
3491        }
3492        self.state.seats.insert(final_seat.key.clone(), final_seat);
3493
3494        let rescue_message = format!(
3495            "magi: operator-selected fix ({}) (uncommitted work)",
3496            self.state.operator_fixes[request_index]
3497                .findings
3498                .iter()
3499                .map(|f| f.id.as_str())
3500                .collect::<Vec<_>>()
3501                .join(", ")
3502        );
3503        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
3504            self.state.note_withheld("fix", &r.withheld);
3505        }
3506        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
3507        fix.committed = after != head_at_request;
3508        git::worktree_remove(&self.state.repo, &fix_worktree)
3509            .await
3510            .ok();
3511
3512        self.state.event(
3513            "fix",
3514            match &fix.failed {
3515                Some(reason) => format!(
3516                    "operator fix: adoption report was lost ({reason}); {}",
3517                    if fix.committed {
3518                        "committed"
3519                    } else {
3520                        "NO new commit"
3521                    }
3522                ),
3523                None => format!(
3524                    "operator fix: {} addressed, {} rejected, {}",
3525                    fix.addressed.len(),
3526                    fix.rejected.len(),
3527                    if fix.committed {
3528                        "committed"
3529                    } else {
3530                        "NO new commit"
3531                    }
3532                ),
3533            },
3534        );
3535
3536        // Every selected finding gets an outcome — never left `Pending` once
3537        // the fixer's own turn is over. A report that never came back at all
3538        // marks every one of them `Unreported`, not silently "not addressed":
3539        // quota, a dropped stream, or an exhausted continuation are gaps in
3540        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
3541        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
3542        for f in &mut self.state.operator_fixes[request_index].findings {
3543            f.outcome = if fix.failed.is_some() {
3544                OperatorFixOutcome::Unreported
3545            } else if fix.addressed.contains(&f.id) {
3546                OperatorFixOutcome::Addressed
3547            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
3548                OperatorFixOutcome::Rejected { why: r.why.clone() }
3549            } else {
3550                OperatorFixOutcome::Unreported
3551            };
3552        }
3553
3554        let committed = fix.committed;
3555        if committed {
3556            self.state.operator_fixes[request_index].result_head = Some(after.clone());
3557        }
3558        self.state.operator_fixes[request_index].fix = Some(fix);
3559        // Saved again now that the fixer's own outcome is final, on top of
3560        // the save right after the request was first pushed above.
3561        self.state.save()?;
3562
3563        if committed {
3564            self.state.event(
3565                "fix",
3566                format!(
3567                    "operator fix committed {}; opening a follow-up review-only run",
3568                    short(&after)
3569                ),
3570            );
3571            match Self::review(&self.state.repo, &winner.branch, self.state.config.clone()).await {
3572                Ok(mut follow_up) => {
3573                    follow_up.state.event(
3574                        "start",
3575                        format!(
3576                            "requested by an operator fix on run {} for finding(s) {}",
3577                            self.state.id,
3578                            self.state.operator_fixes[request_index]
3579                                .findings
3580                                .iter()
3581                                .map(|f| f.id.as_str())
3582                                .collect::<Vec<_>>()
3583                                .join(", "),
3584                        ),
3585                    );
3586                    follow_up.state.save()?;
3587                    let follow_up_id = follow_up.state.id.clone();
3588                    if let Err(e) = follow_up.execute().await {
3589                        self.state.event(
3590                            "fix",
3591                            format!(
3592                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
3593                            ),
3594                        );
3595                    }
3596                    self.state.operator_fixes[request_index].follow_up_review_run =
3597                        Some(follow_up_id);
3598                }
3599                Err(e) => {
3600                    self.state.event(
3601                        "fix",
3602                        format!("committed the fix but could not open a follow-up review: {e:#}"),
3603                    );
3604                }
3605            }
3606            self.state.save()?;
3607        }
3608
3609        Ok(())
3610    }
3611
3612    // --------------------------------------------------------------- review
3613
3614    /// The agent and seat key that fix the winner's tree: the configured
3615    /// fixer, else the winner's own implementer seat, whose conversation
3616    /// continues now that the competition is over. Shared by the review loop
3617    /// and the gate-fix round so both talk to the same seat.
3618    fn fixer_spec(&self, winner: &Candidate) -> (AgentSpec, String) {
3619        match &self.roles.fixer {
3620            Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3621            _ => (
3622                self.state
3623                    .config
3624                    .agent(&winner.agent)
3625                    .cloned()
3626                    .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3627                format!("impl-{}", winner.label),
3628            ),
3629        }
3630    }
3631
3632    async fn review_loop(&mut self) -> Result<()> {
3633        // A base that would not rebase is a person's decision, not a review
3634        // round: nothing here would change the answer, and reviewers and a
3635        // fixer would be spending real budget on a tree that cannot land
3636        // regardless of what they find.
3637        if self
3638            .state
3639            .base_sync
3640            .as_ref()
3641            .is_some_and(|s| s.conflict.is_some())
3642        {
3643            return Ok(());
3644        }
3645        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3646        // agent files with `magi task add` name the run that paid for it. The
3647        // prompt overlay is cloned alongside it because the waves borrow it
3648        // while `self` is mutably borrowed by the node's own bookkeeping.
3649        let run_id = self.state.id.clone();
3650        let prompts = self.state.config.prompts.clone();
3651        let Some(winner) = self.state.winner().cloned() else {
3652            return Ok(());
3653        };
3654        let max_rounds = self.state.config.graph.review_rounds;
3655        // A clean round, an exhausted round budget, or a stalled tree (see
3656        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
3657        // they are recorded — recomputed here, not read off `status`, so a
3658        // reentry into a run that already stopped restates the identical
3659        // verdict instead of silently handing back whatever an earlier node
3660        // in this same walk clobbered `status` to (a solo-candidate
3661        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
3662        // below runs an empty range once the budget is spent, and would
3663        // otherwise fall through without touching `status` at all.
3664        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
3665            self.state.status = status;
3666            self.state.save()?;
3667            return Ok(());
3668        }
3669        self.state.status = RunStatus::Reviewing;
3670        // A last recorded round whose own verification never resolved
3671        // (`ResourceBlocked` — the shared build cache, not the patch) is
3672        // never a concluded round, whatever the round budget says: starting
3673        // a fresh round on top of it would spend a whole new reviewer wave
3674        // re-reading an unchanged patch instead of just retrying the one
3675        // check that actually needs it, and once the budget is spent the
3676        // loop below has nothing left to do at all (its range is empty).
3677        // Retry that check directly instead, exactly the same retry
3678        // `stop_reviewing` already does for its own catch-up case.
3679        if self
3680            .state
3681            .reviews
3682            .last()
3683            .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
3684        {
3685            let shell = self.state.config.shell();
3686            return self
3687                .stop_reviewing(
3688                    "the last round's own verification never resolved",
3689                    &shell,
3690                    &winner.worktree,
3691                )
3692                .await;
3693        }
3694
3695        let repo = self.state.repo.clone();
3696        let root = self.state.worktree_root();
3697        let language = self.state.config.graph.language.clone();
3698        let sessions = self.state.config.graph.sessions;
3699        let artifacts = agent::artifacts_dir(&self.state.dir());
3700        let base = self.landing_base();
3701        let base_short = short(&base);
3702        let reviewers = self.roles.reviewers.clone();
3703        let shell = self.state.config.shell();
3704
3705        for round in (self.state.reviews.len() + 1)..=max_rounds {
3706            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3707            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
3708            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
3709            // The prior round's own record, already persisted — never a
3710            // hand-carried variable of just its failing output: that is
3711            // exactly what let a round's e2e result drift out of sync with
3712            // which commit it was actually about (see `SCHEMA`'s doc for
3713            // schema 8). Judged against `head`, the commit reviewers are
3714            // about to look at now, so the summary always reads as "an
3715            // earlier head" here — this round's own patch has not been
3716            // checked yet.
3717            let prev_verification = self
3718                .state
3719                .reviews
3720                .last()
3721                .and_then(|r| r.verification_summary(&head));
3722
3723            // Each reviewer gets its own detached checkout of exactly this
3724            // commit: nobody can perturb the winner's tree, and the fixer can
3725            // keep working without racing a reviewer.
3726            let mut jobs = Vec::new();
3727            for (r, spec) in reviewers.iter().cloned().enumerate() {
3728                let wt = root.join(format!("review-{}", r + 1));
3729                if wt.exists() {
3730                    git::reset_detached(&wt, &head).await?;
3731                } else {
3732                    git::worktree_add_detached(&repo, &wt, &head).await?;
3733                }
3734                let seat_key = format!("review-{}", r + 1);
3735                let seat = self.seat(&seat_key, &spec.id);
3736                jobs.push(SeatJob {
3737                    prompt: prompt::review(&prompt::ReviewCtx {
3738                        instruction: &self.state.instruction,
3739                        branch: &winner.branch,
3740                        base_short: &base_short,
3741                        stat: &stat,
3742                        patch: &patch,
3743                        verification: prev_verification.as_ref(),
3744                        reviewers: reviewers.len(),
3745                        round,
3746                        rounds: max_rounds,
3747                        // A review-only run has no rankings, so nothing
3748                        // competed for this patch and the reviewer is told so.
3749                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
3750                        lens: Lens::for_seat(r),
3751                        language: &language,
3752                    }),
3753                    spec,
3754                    seat,
3755                    cwd: wt,
3756                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
3757                    allow_write: false,
3758                    sessions,
3759                    artifacts: artifacts.clone(),
3760                    stem: format!("review-{round}-{}", r + 1),
3761                });
3762            }
3763
3764            self.state.event(
3765                "review",
3766                format!(
3767                    "round {round}: {} reviewers on {}",
3768                    jobs.len(),
3769                    short(&head)
3770                ),
3771            );
3772            let mut quota_losses = Vec::new();
3773            let review_retries = self.state.config.graph.retries;
3774            let review_cache = self.state.config.cache_dir();
3775            let ctx = WaveCtx {
3776                run: &run_id,
3777                node: "review",
3778                prompts: &prompts,
3779                cache: review_cache.as_deref(),
3780                round: Some(round),
3781            };
3782            let results = ask_json_wave::<Review>(
3783                jobs,
3784                Arc::clone(&self.sem),
3785                review_retries,
3786                &ctx,
3787                &mut quota_losses,
3788                &mut self.state,
3789                &|_: &Review| Ok(()),
3790            )
3791            .await;
3792            // Counted before the move below: how many of *this* round's
3793            // reviewer seats were lost to their own rate limit, as opposed to
3794            // a crash, a timeout, or unparsable output — see `round_is_clean`.
3795            let round_quota_missing = quota_losses.len();
3796            self.state.quota.extend(quota_losses);
3797
3798            let mut records = Vec::new();
3799            let mut all_findings = Vec::new();
3800            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
3801                let agent_id = seat.agent.clone();
3802                self.state.seats.insert(seat.key.clone(), seat);
3803                let mut record = ReviewRecord {
3804                    reviewer: r + 1,
3805                    agent: agent_id,
3806                    summary: String::new(),
3807                    findings: Vec::new(),
3808                    vote: None,
3809                    failed: None,
3810                    duration_ms: 0,
3811                    // Set for both outcomes: `failed: Some(_)` with
3812                    // `attempts > 0` is a seat every retry still lost, not a
3813                    // recovered one — only `failed: None` with `attempts > 0`
3814                    // reads as "answered after a nudge" (see this field's own
3815                    // doc).
3816                    attempts,
3817                };
3818                match res {
3819                    Ok((review, out)) => {
3820                        // Sanitized here, at the point every other piece of
3821                        // agent prose in this file is (candidate summaries,
3822                        // deliberation turns, vote reasons): a reviewer's own
3823                        // words are the one thing about it that could name
3824                        // it, and reconsideration below broadcasts this same
3825                        // summary and these same findings to every other
3826                        // seat on the panel.
3827                        record.summary =
3828                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
3829                        record.vote = Some(review.vote);
3830                        record.duration_ms = out.duration_ms;
3831                        for (n, mut f) in review.findings.into_iter().enumerate() {
3832                            // ids are magi's, never the agent's: the fixer's
3833                            // adoption report is keyed by them.
3834                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
3835                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
3836                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
3837                            // `file` is agent-supplied prose too, never
3838                            // checked against the real tree — the same
3839                            // exposure `title`/`detail` above have, just in
3840                            // a field easy to forget because it looks like a
3841                            // path rather than free text.
3842                            f.file = f
3843                                .file
3844                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
3845                            all_findings.push(f.clone());
3846                            record.findings.push(f);
3847                        }
3848                        self.state.event(
3849                            "review",
3850                            format!(
3851                                "round {round}: reviewer {} voted {} with {} finding(s)",
3852                                r + 1,
3853                                review.vote.label(),
3854                                record.findings.len()
3855                            ),
3856                        );
3857                    }
3858                    Err(e) => {
3859                        record.failed = Some(e.to_string());
3860                        self.state.event(
3861                            "review",
3862                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
3863                        );
3864                    }
3865                }
3866                records.push(record);
3867            }
3868
3869            // Tally the round's votes and, if they split, spend the one
3870            // round of reconsideration the split -> deliberate -> revote
3871            // shape `judge`/`vote` use for the panel, sized down to what a
3872            // read-only review round can afford: one round, and a revote
3873            // rather than an argument, because the panel already wrote its
3874            // reasoning down as findings the first time around.
3875            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
3876            let vote_split =
3877                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
3878            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
3879            if vote_split {
3880                self.state.event(
3881                    "review",
3882                    format!(
3883                        "round {round}: votes split ({}) — one round of reconsideration",
3884                        initial_votes
3885                            .iter()
3886                            .map(|v| v.label())
3887                            .collect::<Vec<_>>()
3888                            .join(", ")
3889                    ),
3890                );
3891                // Seats read every seat's findings and votes, still numbered
3892                // and never named — the same anonymity `review` itself keeps.
3893                let panel: Vec<ReviewSeatReport<'_>> = records
3894                    .iter()
3895                    .filter_map(|r| {
3896                        r.vote.map(|vote| ReviewSeatReport {
3897                            reviewer: r.reviewer,
3898                            vote,
3899                            summary: &r.summary,
3900                            findings: &r.findings,
3901                        })
3902                    })
3903                    .collect();
3904
3905                let mut jobs = Vec::new();
3906                let mut seats_at = Vec::new();
3907                for (r, spec) in reviewers.iter().cloned().enumerate() {
3908                    // A seat with no initial vote has nothing to reconsider
3909                    // from and stays absent, the same as it stayed absent
3910                    // from `panel` above.
3911                    if records[r].vote.is_none() {
3912                        continue;
3913                    }
3914                    let wt = root.join(format!("review-{}", r + 1));
3915                    let seat_key = format!("review-{}", r + 1);
3916                    let seat = self.seat(&seat_key, &spec.id);
3917                    // A seat with no live session has already forgotten the
3918                    // initial review's prompt — restate the patch it is
3919                    // voting on, the same as `deliberate`/`vote` do for a
3920                    // judge in the same position.
3921                    let patch_ctx = if has_context(&spec, &seat, sessions) {
3922                        None
3923                    } else {
3924                        Some(ReviewPatch {
3925                            branch: &winner.branch,
3926                            base_short: &base_short,
3927                            stat: &stat,
3928                            patch: &patch,
3929                        })
3930                    };
3931                    let prompt = prompt::review_reconsider(&ReviewReconsiderCtx {
3932                        instruction: &self.state.instruction,
3933                        reviewer: r + 1,
3934                        lens: Lens::for_seat(r),
3935                        panel: &panel,
3936                        patch: patch_ctx,
3937                        round,
3938                        rounds: max_rounds,
3939                        language: &language,
3940                    });
3941                    jobs.push(SeatJob {
3942                        prompt,
3943                        spec,
3944                        seat,
3945                        cwd: wt,
3946                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
3947                        allow_write: false,
3948                        sessions,
3949                        artifacts: artifacts.clone(),
3950                        stem: format!("review-{round}-reconsider-{}", r + 1),
3951                    });
3952                    seats_at.push(r);
3953                }
3954
3955                let mut recon_quota_losses = Vec::new();
3956                let recon_cache = self.state.config.cache_dir();
3957                let recon_ctx = WaveCtx {
3958                    run: &run_id,
3959                    node: "review",
3960                    prompts: &prompts,
3961                    cache: recon_cache.as_deref(),
3962                    round: Some(round),
3963                };
3964                let recon_results = ask_json_wave::<ReviewRevote>(
3965                    jobs,
3966                    Arc::clone(&self.sem),
3967                    review_retries,
3968                    &recon_ctx,
3969                    &mut recon_quota_losses,
3970                    &mut self.state,
3971                    &|_: &ReviewRevote| Ok(()),
3972                )
3973                .await;
3974                self.state.quota.extend(recon_quota_losses);
3975
3976                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
3977                    let agent_id = seat.agent.clone();
3978                    self.state.seats.insert(seat.key.clone(), seat);
3979                    let mut rec = ReviewRevoteRecord {
3980                        reviewer: r + 1,
3981                        agent: agent_id,
3982                        vote: None,
3983                        reason: String::new(),
3984                        failed: None,
3985                    };
3986                    match res {
3987                        Ok((rv, _)) => {
3988                            rec.vote = Some(rv.vote);
3989                            rec.reason =
3990                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
3991                            self.state.event(
3992                                "review",
3993                                format!(
3994                                    "round {round}: reviewer {} revoted {}",
3995                                    r + 1,
3996                                    rv.vote.label()
3997                                ),
3998                            );
3999                        }
4000                        Err(e) => {
4001                            rec.failed = Some(e.to_string());
4002                            self.state.event(
4003                                "review",
4004                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
4005                            );
4006                        }
4007                    }
4008                    reconsideration.push(rec);
4009                }
4010            } else if initial_votes.len() > 1 {
4011                self.state.event(
4012                    "review",
4013                    format!(
4014                        "round {round}: votes agreed ({}) — no reconsideration",
4015                        initial_votes[0].label()
4016                    ),
4017                );
4018            }
4019
4020            // The final vote per seat is its revote where reconsideration
4021            // ran and answered, its initial vote otherwise — the same
4022            // fallback `tally` uses for a judge whose private vote failed.
4023            let final_votes: Vec<ReviewVote> = records
4024                .iter()
4025                .filter_map(|r| {
4026                    reconsideration
4027                        .iter()
4028                        .find(|rv| rv.reviewer == r.reviewer)
4029                        .and_then(|rv| rv.vote)
4030                        .or(r.vote)
4031                })
4032                .collect();
4033            let round_verdict = ReviewVote::worst(final_votes);
4034
4035            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
4036            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4037            // A round that already has a blocking finding and a round left to
4038            // try is going back to the fixer no matter what `verify.e2e`
4039            // says, so running it first only spends the loop's slowest step
4040            // (minutes, for a Rust repo's full test suite) on a head about
4041            // to be rewritten. Deferred, never skipped: `verify.e2e` still
4042            // runs once a round has no blocking findings left (see
4043            // `round_is_clean`, which a deferred — empty — `e2e` can never
4044            // satisfy since `blocking` is nonzero whenever this branch is
4045            // taken), and `stop_reviewing` forces a real run before it will
4046            // ever read a deferred round as green.
4047            let defer_e2e =
4048                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
4049            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
4050                let reason =
4051                    format!("{blocking} blocking finding(s) already required a fix this round");
4052                self.state.event(
4053                    "verify",
4054                    format!(
4055                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
4056                         {}); it will run once a round has none left",
4057                        short(&head)
4058                    ),
4059                );
4060                (Vec::new(), false, true, Some(reason))
4061            } else {
4062                let e2e_commands = self.state.config.verify.e2e.clone();
4063                let cache_dir = self.state.config.cache_dir();
4064                let context = format!("round {round}");
4065                let (e2e, verify_retried) = with_cache_lease(
4066                    &mut self.state,
4067                    cache_dir.as_deref(),
4068                    "e2e",
4069                    "e2e",
4070                    &winner.worktree,
4071                    &head,
4072                    verify_timeout,
4073                    &context,
4074                    |state, budget| {
4075                        let shell = shell.clone();
4076                        let e2e_commands = e2e_commands.clone();
4077                        let worktree = winner.worktree.clone();
4078                        let context = context.clone();
4079                        async move {
4080                            run_e2e_with_retry(
4081                                state,
4082                                &shell,
4083                                &e2e_commands,
4084                                &worktree,
4085                                budget,
4086                                &context,
4087                            )
4088                            .await
4089                        }
4090                    },
4091                )
4092                .await;
4093                (e2e, verify_retried, false, None)
4094            };
4095
4096            let expected = records.len();
4097            let answered = records.iter().filter(|r| r.failed.is_none()).count();
4098            let incomplete = answered < expected;
4099            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
4100            let policy = self.state.config.graph.incomplete_review;
4101            let clean = round_is_clean(
4102                blocking,
4103                e2e_ok,
4104                answered,
4105                expected,
4106                round_quota_missing,
4107                policy,
4108            );
4109
4110            let mut round_record = ReviewRound {
4111                round,
4112                head: head.clone(),
4113                verified_head: None,
4114                verified_at: None,
4115                reviews: records,
4116                e2e,
4117                verify_retried,
4118                e2e_deferred,
4119                e2e_defer_reason,
4120                fix: None,
4121                blocking,
4122                answered,
4123                expected,
4124                clean,
4125                progressed: false,
4126                vote_split,
4127                reconsideration,
4128                verdict: round_verdict,
4129            };
4130            // Which commit and when magi actually attempted to check —
4131            // known the moment a command was dispatched against `head`,
4132            // whether or not it finished: a resource-blocked attempt still
4133            // targeted a specific commit at a specific time, and leaving
4134            // that unrecorded is exactly what made `verification_summary`
4135            // report a fresh attempt as "commit unknown ... recorded before
4136            // this was tracked", indistinguishable from a genuinely old,
4137            // untracked record. Only a deferred or unconfigured round never
4138            // ran at all and has nothing to record — see
4139            // `ReviewRound::verified_head`'s own doc.
4140            if !matches!(
4141                round_record.e2e_status(),
4142                E2eStatus::Deferred | E2eStatus::NotConfigured
4143            ) {
4144                round_record.verified_head = Some(head.clone());
4145                round_record.verified_at = Some(Timestamp::now());
4146            }
4147            let this_round_verification = round_record.verification_summary(&head);
4148
4149            if incomplete {
4150                let missing: Vec<String> = round_record
4151                    .reviews
4152                    .iter()
4153                    .filter(|r| r.failed.is_some())
4154                    .map(|r| format!("review-{}", r.reviewer))
4155                    .collect();
4156                self.state.event(
4157                    "review",
4158                    format!(
4159                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
4160                        missing.join(", ")
4161                    ),
4162                );
4163            }
4164
4165            if clean {
4166                self.state.event(
4167                    "review",
4168                    if incomplete && policy == IncompleteReviewPolicy::Warn {
4169                        format!(
4170                            "round {round}: clean (warn policy, incomplete panel) — no \
4171                             blocking findings from the seats that answered, verification green"
4172                        )
4173                    } else if incomplete {
4174                        format!(
4175                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
4176                             quorum) — no blocking findings from the seats that answered, \
4177                             verification green",
4178                            expected - answered
4179                        )
4180                    } else {
4181                        format!("round {round}: clean — no blocking findings, verification green")
4182                    },
4183                );
4184                self.state.reviews.push(round_record);
4185                self.state.status = RunStatus::Gating;
4186                self.state.save()?;
4187                return Ok(());
4188            }
4189
4190            // Nothing was raised and verification passed, but not every seat
4191            // answered and `round_is_clean` still refused to call it clean —
4192            // either a seat is missing for a reason other than its own quota
4193            // (a crash, a timeout, unparsable output — worth another try), or
4194            // every seat that could have answered lost its quota and nobody
4195            // is left to decide on: re-review rather than send the fixer
4196            // after a round with nothing to fix.
4197            if incomplete && blocking == 0 && e2e_ok {
4198                self.state.reviews.push(round_record);
4199                self.state.save()?;
4200                if round == max_rounds {
4201                    self.state.status = RunStatus::Blocked;
4202                    self.state.event(
4203                        "review",
4204                        format!(
4205                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
4206                             refusing to call it clean",
4207                            expected - answered
4208                        ),
4209                    );
4210                    return Ok(());
4211                }
4212                continue;
4213            }
4214
4215            // Nothing for the fixer to act on (`blocking == 0`) and the only
4216            // reason this round is not clean is that magi itself never got
4217            // a command to run — the shared build cache, not the patch (see
4218            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
4219            // the fixer would invite a change to appease contention that has
4220            // nothing to do with the diff, and would leave this attempt
4221            // sitting in the next round's prompt as if it were about an
4222            // earlier, superseded commit rather than what it actually is:
4223            // the same head, still waiting to be checked. Wait for it the
4224            // same way the final round's own contention is already handled,
4225            // whatever round this happens to be.
4226            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
4227                self.state.reviews.push(round_record);
4228                return self
4229                    .stop_reviewing(
4230                        "the round's own verification could not run",
4231                        &shell,
4232                        &winner.worktree,
4233                    )
4234                    .await;
4235            }
4236
4237            if round == max_rounds {
4238                self.state.reviews.push(round_record);
4239                return self
4240                    .stop_reviewing(
4241                        &format!(
4242                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
4243                        ),
4244                        &shell,
4245                        &winner.worktree,
4246                    )
4247                    .await;
4248            }
4249
4250            // Fix. The winner's own implementer seat continues its conversation:
4251            // the competition is over, so context is pure benefit now.
4252            let (fix_spec, fix_seat_key) = self.fixer_spec(&winner);
4253            let seat = self.seat(&fix_seat_key, &fix_spec.id);
4254            let blocking_findings: Vec<_> = all_findings
4255                .iter()
4256                .filter(|f| f.severity.blocks())
4257                .cloned()
4258                .collect();
4259            let job = SeatJob {
4260                prompt: prompt::fix(
4261                    &self.state.instruction,
4262                    &blocking_findings,
4263                    this_round_verification.as_ref(),
4264                    round,
4265                    max_rounds,
4266                    &language,
4267                ),
4268                spec: fix_spec.clone(),
4269                seat,
4270                cwd: winner.worktree.clone(),
4271                timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
4272                allow_write: true,
4273                sessions,
4274                artifacts: artifacts.clone(),
4275                stem: format!("fix-{round}"),
4276            };
4277            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
4278            let cache = self.state.config.cache_dir();
4279            let ctx = WaveCtx {
4280                run: &run_id,
4281                node: "fix",
4282                prompts: &prompts,
4283                cache: cache.as_deref(),
4284                round: Some(round),
4285            };
4286            let (seat, out) =
4287                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4288            let agent_id = seat.agent.clone();
4289
4290            let mut fix = FixRecord {
4291                agent: agent_id,
4292                addressed: Vec::new(),
4293                rejected: Vec::new(),
4294                notes: String::new(),
4295                committed: false,
4296                failed: None,
4297                duration_ms: 0,
4298                continuation: None,
4299            };
4300            let mut continuation = ContinuationRecord::not_needed();
4301            let mut final_seat = seat.clone();
4302            match out {
4303                AgentOutcome::Ok(o) => {
4304                    fix.duration_ms = o.duration_ms;
4305                    let parsed = verdict::extract_json::<FixReport>(&o.text);
4306                    // A parsed report standing next to a command this same
4307                    // reply's own CLI never confirmed the exit status of is
4308                    // not a resolved answer — the identical `CommandEvidence`
4309                    // `state.jobs` renders, read here instead of only on
4310                    // display, per the completion judgment and the shown
4311                    // record needing to agree.
4312                    let incomplete_reason = match &parsed {
4313                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4314                            "the reply parsed, but it reported a command whose own CLI never \
4315                             confirmed an exit status"
4316                                .to_owned(),
4317                        ),
4318                        Ok(_) => None,
4319                        Err(e) => Some(e.to_string()),
4320                    };
4321                    match incomplete_reason {
4322                        None => {
4323                            let report = parsed.expect("checked Ok above");
4324                            fix.addressed = report.addressed;
4325                            fix.rejected = report.rejected;
4326                            fix.notes =
4327                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
4328                        }
4329                        Some(reason) => {
4330                            let (resumed_seat, resolved, failure, cont) = self
4331                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
4332                                .await;
4333                            fix.duration_ms += cont.cumulative_wait_ms;
4334                            continuation = cont;
4335                            final_seat = resumed_seat;
4336                            match resolved {
4337                                Some(report) => {
4338                                    fix.addressed = report.addressed;
4339                                    fix.rejected = report.rejected;
4340                                    fix.notes = blind::sanitize_prose(
4341                                        &report.notes,
4342                                        &self.state.config.blind,
4343                                    );
4344                                }
4345                                None => fix.failed = failure,
4346                            }
4347                        }
4348                    }
4349                }
4350                // The CLI's raw error JSON is not a fix report to parse.
4351                AgentOutcome::Dropped(o) => {
4352                    fix.duration_ms = o.duration_ms;
4353                    let why = o
4354                        .dropped
4355                        .as_ref()
4356                        .map(|d| d.why.as_str())
4357                        .unwrap_or("the CLI ended the stream without delivering its answer");
4358                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4359                }
4360                AgentOutcome::Quota(o) => {
4361                    self.state.quota.push(QuotaLoss {
4362                        seat: final_seat.key.clone(),
4363                        node: "fix".to_owned(),
4364                        at: Timestamp::now(),
4365                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4366                    });
4367                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4368                }
4369                AgentOutcome::Failed(e) => fix.failed = Some(e),
4370            }
4371            fix.continuation = Some(continuation);
4372            self.state.seats.insert(final_seat.key.clone(), final_seat);
4373            if let Ok(r) = git::rescue_commit(
4374                &winner.worktree,
4375                &format!("magi: review round {round} fixes (uncommitted work)"),
4376            )
4377            .await
4378            {
4379                self.state.note_withheld("fix", &r.withheld);
4380            }
4381            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
4382            fix.committed = after != before;
4383            // Judged by what `git` says moved against base, never by the
4384            // fixer's own `addressed`/`rejected` count — see
4385            // `ReviewRound::progressed`. Propagated with `?`, the same as the
4386            // `patch` snapshot above: swallowing this error would default
4387            // `diff_after` to empty, which almost always differs from a
4388            // non-empty `patch` and reads as "progressed" — exactly backwards
4389            // for a `git` failure the stagnation check cannot see through.
4390            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
4391            let progressed = diff_after != patch;
4392            let commit_note = if fix.committed {
4393                "committed"
4394            } else {
4395                "NO new commit"
4396            };
4397            let tree_note = if progressed {
4398                "changed vs base"
4399            } else {
4400                "unchanged vs base"
4401            };
4402            self.state.event(
4403                "fix",
4404                match &fix.failed {
4405                    // Distinct on purpose from "0 addressed, 0 rejected": the
4406                    // fixer's own diff still landed (blocking counts do keep
4407                    // falling round over round), only its adoption report did
4408                    // not come back, so this must never read like every
4409                    // finding was reviewed and declined.
4410                    Some(reason) => {
4411                        format!(
4412                            "round {round}: fixer's adoption report was lost ({reason}); \
4413                             {commit_note}, tree {tree_note}"
4414                        )
4415                    }
4416                    None => format!(
4417                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
4418                         {tree_note}{}",
4419                        fix.addressed.len(),
4420                        fix.rejected.len(),
4421                        if continuation.outcome == ContinuationOutcome::Resumed {
4422                            format!(
4423                                " (adoption report recovered after {} continuation(s))",
4424                                continuation.attempts
4425                            )
4426                        } else {
4427                            String::new()
4428                        },
4429                    ),
4430                },
4431            );
4432            round_record.fix = Some(fix);
4433            round_record.progressed = progressed;
4434            self.state.reviews.push(round_record);
4435            self.state.save()?;
4436
4437            // The fixer's own report never came back this round, even after
4438            // `continue_fix_report`'s own budget was spent on it — not an
4439            // ordinary "no report" (dropped stream, quota, plain failure),
4440            // which already reads that way and is left to the existing round
4441            // budget. Stopping here, rather than opening another round, is
4442            // what keeps a next reviewer/fixer wave from ever being
4443            // dispatched onto `winner.worktree` while whatever the seat's
4444            // last call may still have running there is unaccounted for: no
4445            // process liveness check exists (and none is being added — see
4446            // AGENTS.md/this task's own scope), so the only way to honour
4447            // "nothing starts before a valid report returns" is to not start
4448            // anything further on this worktree from this run at all.
4449            if matches!(
4450                continuation.outcome,
4451                ContinuationOutcome::Exhausted
4452                    | ContinuationOutcome::QuotaLost
4453                    | ContinuationOutcome::NoSession
4454            ) {
4455                return self
4456                    .stop_reviewing(
4457                        "the fixer's adoption report never came back, even after resuming its \
4458                         own seat; refusing to start another round against the same worktree \
4459                         while that is unresolved",
4460                        &shell,
4461                        &winner.worktree,
4462                    )
4463                    .await;
4464            }
4465
4466            let streak = self
4467                .state
4468                .reviews
4469                .iter()
4470                .rev()
4471                .take_while(|r| !r.progressed)
4472                .count();
4473            if streak >= STAGNANT_LIMIT {
4474                return self
4475                    .stop_reviewing(
4476                        &format!(
4477                            "the tree has not moved against base for {streak} round(s) in a row"
4478                        ),
4479                        &shell,
4480                        &winner.worktree,
4481                    )
4482                    .await;
4483            }
4484        }
4485        Ok(())
4486    }
4487
4488    /// Decide, from the last recorded round's own verification, whether
4489    /// stopping the review loop is a hand-off or a genuine block.
4490    ///
4491    /// Called once the loop has given up trying — the round budget is spent,
4492    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
4493    /// findings still open, never while a round is still clean or the
4494    /// incomplete-panel case handled inline above. Gate and e2e are facts
4495    /// about the tree; a lingering review finding is an opinion, and this
4496    /// workload's own `magi stats` puts reviewer precision low enough
4497    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
4498    /// must not by itself stand between a green, verified change and the
4499    /// human who decides what to do with it. A red e2e is not an opinion, so
4500    /// that case still blocks, with the failing command and a tail of its
4501    /// output recorded here rather than left in `run.json` for someone to go
4502    /// find.
4503    ///
4504    /// A round that deferred its own e2e (see [`Config::graph`]'s
4505    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
4506    /// only because nothing ran, and treating an empty list as a passing one
4507    /// here is exactly the "deferred painted green" bug this function exists
4508    /// to not have. When the last round's own verification never resolved —
4509    /// deferred on purpose, or a real attempt the shared build cache blocked
4510    /// — this makes (or retries) the real run, on the actual worktree this
4511    /// loop is about to stop touching, before deciding anything. A
4512    /// resource-blocked attempt is likewise never read as either green or
4513    /// red: it is evidence about the machine, not the patch (see
4514    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
4515    /// blocked cache leaves this call without deciding rather than guessing
4516    /// — the caller retries on a later reentry.
4517    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
4518        let round_idx = self.state.reviews.len() - 1;
4519        // A deferred round and a resource-blocked one are the same shape
4520        // here: neither has a real result yet, and both get one more
4521        // attempt. Read off `e2e_status` — the single source for this —
4522        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
4523        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
4524        // still retries instead of being read as a settled result the
4525        // instant it stops being empty.
4526        let needs_catchup_run = matches!(
4527            self.state.reviews[round_idx].e2e_status(),
4528            E2eStatus::Deferred | E2eStatus::ResourceBlocked
4529        );
4530        if needs_catchup_run {
4531            let round = self.state.reviews[round_idx].round;
4532            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4533            let commands = self.state.config.verify.e2e.clone();
4534            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
4535            let cache_dir = self.state.config.cache_dir();
4536            let context = format!(
4537                "round {round}: verification unresolved, catching up before the final decision"
4538            );
4539            let (outcomes, verify_retried) = with_cache_lease(
4540                &mut self.state,
4541                cache_dir.as_deref(),
4542                "e2e",
4543                "e2e",
4544                worktree,
4545                &attempted_head,
4546                timeout,
4547                &context,
4548                |state, budget| {
4549                    let shell = shell.to_vec();
4550                    let commands = commands.clone();
4551                    let context = context.clone();
4552                    async move {
4553                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
4554                            .await
4555                    }
4556                },
4557            )
4558            .await;
4559            let last = &mut self.state.reviews[round_idx];
4560            last.e2e = outcomes;
4561            last.verify_retried = verify_retried;
4562            // Always the commit and time this attempt actually targeted,
4563            // whether or not it happens to equal the reviewed `head` and
4564            // whether or not a command finished — see
4565            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
4566            // attempt is recorded too, so a later reader sees "attempted
4567            // again at T2" rather than silence.
4568            last.verified_head = Some(attempted_head);
4569            last.verified_at = Some(Timestamp::now());
4570            if verify_inconclusive(&last.e2e) {
4571                // Still not a real result: `e2e_deferred` is left exactly
4572                // as it was, so `needs_catchup_run` above reads
4573                // `ResourceBlocked` (via `e2e_status`, which checks
4574                // `resource_blocked` before `e2e_deferred`) and retries
4575                // again on the next reentry, rather than recording
4576                // contention as a red e2e and blocking the run on it.
4577                self.state.save()?;
4578                return Ok(());
4579            }
4580            last.e2e_deferred = false;
4581        }
4582        let last = &self.state.reviews[round_idx];
4583        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
4584
4585        match last.e2e_status() {
4586            E2eStatus::Failed => {
4587                let red: Vec<String> = last
4588                    .e2e
4589                    .iter()
4590                    .filter(|o| !o.ok())
4591                    .map(|o| {
4592                        format!(
4593                            "`{}` -> {:?}\n{}",
4594                            o.command,
4595                            o.code,
4596                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
4597                        )
4598                    })
4599                    .collect();
4600                self.state
4601                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
4602                self.state.status = RunStatus::Blocked;
4603            }
4604            // `needs_catchup_run` above already retried once this call; if
4605            // it is still blocked, this is magi's own admission it could
4606            // not get a command to run, never a verdict on the patch — the
4607            // run is left exactly where a later reentry can retry again.
4608            E2eStatus::ResourceBlocked => {
4609                self.state.event(
4610                    "review",
4611                    format!(
4612                        "{why}; e2e could not run (shared build cache unavailable); not \
4613                         deciding yet"
4614                    ),
4615                );
4616            }
4617            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
4618                self.state.event(
4619                    "review",
4620                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
4621                );
4622                self.state.status = RunStatus::Gating;
4623            }
4624        }
4625        self.state.save()?;
4626        Ok(())
4627    }
4628
4629    // ----------------------------------------------------------------- gate
4630
4631    async fn gate(&mut self) -> Result<()> {
4632        // Judged by the review record itself, not by `status`: a solo
4633        // candidate's `judge`/`deliberate` skip rewrites `status` on every
4634        // reentry (see `judge`), and trusting it here is exactly how a run
4635        // that exhausted its review budget got gated and merged a second
4636        // time around. `review_conclusion` recomputes the review loop's own
4637        // verdict from the round records themselves — `Gating` for a clean
4638        // round or a hand-off (see `stop_reviewing`), anything else means the
4639        // loop is still going or genuinely blocked.
4640        // A base the winner could not be replayed onto is a decision, not a
4641        // round: there is no landing tree to gate. Read as its own record for
4642        // the same reason the review verdict is.
4643        if self.state.status == RunStatus::Failed
4644            || self
4645                .state
4646                .base_sync
4647                .as_ref()
4648                .is_some_and(|s| s.conflict.is_some())
4649            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
4650                != Some(RunStatus::Gating)
4651        {
4652            return Ok(());
4653        }
4654        if self.state.gate_ran {
4655            // `review_loop` derives its conclusion from the clean review
4656            // record on every reentry and therefore puts a completed run back
4657            // in `Gating`. A recorded gate is a stronger, terminal fact:
4658            // retain its original command output (or lack of any, for a repo
4659            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
4660            // doc) and restore `Blocked` on a real failure rather than
4661            // pretending the command is still running or running it a second
4662            // time. `gate_ran == false` remains the only shape — unattempted,
4663            // or a resource-blocked retry — that may still need to execute a
4664            // command.
4665            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
4666                self.state.status = RunStatus::Blocked;
4667                self.state.save()?;
4668            }
4669            return Ok(());
4670        }
4671        let Some(winner) = self.state.winner().cloned() else {
4672            return Ok(());
4673        };
4674        self.state.status = RunStatus::Gating;
4675        let mut outcomes = self.run_gate(&winner).await?;
4676        loop {
4677            // A resource-blocked outcome means the gate command never actually
4678            // ran - the shared build cache could not be acquired or confirmed
4679            // fresh in time - which is evidence about the machine, not about
4680            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
4681            // Recording it as a red gate would mark a run `Blocked` on nothing
4682            // but contention magi has already logged; leaving `self.state.gate`
4683            // empty and `self.state.gate_ran` false instead keeps the shape
4684            // this function already treats as "still needs to run" (see the
4685            // early-return above), so the next call retries the command
4686            // rather than concluding anything.
4687            if verify_inconclusive(&outcomes) {
4688                self.state.save()?;
4689                return Ok(());
4690            }
4691            if outcomes.iter().all(CommandOutcome::ok) {
4692                break;
4693            }
4694            match self.gate_fix_round(&winner, &outcomes).await? {
4695                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
4696                GateFix::Stop => break,
4697                GateFix::Defer => {
4698                    self.state.save()?;
4699                    return Ok(());
4700                }
4701            }
4702        }
4703        let passed = outcomes.iter().all(CommandOutcome::ok);
4704        self.state.gate = outcomes;
4705        self.state.gate_ran = true;
4706        if !passed {
4707            self.state.status = RunStatus::Blocked;
4708            let spent = self.state.gate_fixes.len();
4709            self.state.event(
4710                "gate",
4711                if spent == 0 {
4712                    "gate failed; not merging".to_owned()
4713                } else {
4714                    format!("gate failed after {spent} gate-fix round(s); not merging")
4715                },
4716            );
4717        }
4718        self.state.save()?;
4719        Ok(())
4720    }
4721
4722    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
4723    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
4724    /// after review is clean and never on a candidate awaiting judging.
4725    ///
4726    /// Never fails the run: a non-zero exit or timeout is a warning and a
4727    /// recorded outcome, and the gate remains the single arbiter. Nothing
4728    /// configured means nothing happens - no event, no commit. `commit_all`
4729    /// commits any leftover change under the neutral identity and returns
4730    /// `false` when the tree is clean, so no empty commit is ever made.
4731    async fn run_pre_gate(&mut self, winner: &Candidate) {
4732        let commands = self.state.config.verify.pre_gate.clone();
4733        if commands.is_empty() {
4734            return;
4735        }
4736        let shell = self.state.config.shell();
4737        let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4738        let (outcomes, _) = run_commands(
4739            &mut self.state,
4740            "pre_gate",
4741            "pre_gate",
4742            0,
4743            &shell,
4744            &commands,
4745            &winner.worktree,
4746            timeout,
4747        )
4748        .await;
4749        for o in &outcomes {
4750            if !o.ok() {
4751                tracing::warn!(
4752                    "pre_gate `{}` failed ({:?}); the gate decides",
4753                    o.command,
4754                    o.code
4755                );
4756            }
4757            self.state.event(
4758                "pre_gate",
4759                format!(
4760                    "`{}` -> {}",
4761                    o.command,
4762                    if o.ok() {
4763                        "pass".to_owned()
4764                    } else {
4765                        format!(
4766                            "FAIL ({:?})\n{}",
4767                            o.code,
4768                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
4769                        )
4770                    }
4771                ),
4772            );
4773        }
4774        self.state.pre_gate = outcomes;
4775        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
4776            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
4777                Ok(head) => {
4778                    self.state
4779                        .event("pre_gate", format!("committed mechanical fixes ({head})"));
4780                    self.state.pre_gate_commit = Some(head);
4781                }
4782                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
4783            },
4784            Ok(false) => {}
4785            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
4786        }
4787        if let Err(e) = self.state.save() {
4788            tracing::warn!("could not persist the pre_gate record: {e:#}");
4789        }
4790    }
4791
4792    /// Run `verify.gate` once against the winner's current tree, logging one
4793    /// event per command. Empty when nothing is configured.
4794    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
4795        self.run_pre_gate(winner).await;
4796        let shell = self.state.config.shell();
4797        let gate_commands = self.state.config.verify.gate.clone();
4798        // Zero commands has nothing to run and nothing that could touch the
4799        // shared build cache, so it never needs a lease: `Config::cache_dir`
4800        // is derived from `verify.e2e` too, so a repo with no `verify.gate`
4801        // commands but a `CARGO_TARGET_DIR`-using `verify.e2e` would
4802        // otherwise queue behind an unrelated run's lease and come back
4803        // resource-blocked - `gate_ran` would stay false on nothing but
4804        // cache contention, for a step that had nothing to check in the
4805        // first place.
4806        let outcomes = if gate_commands.is_empty() {
4807            Vec::new()
4808        } else {
4809            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4810            let cache_dir = self.state.config.cache_dir();
4811            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4812            let (outcomes, _) = with_cache_lease(
4813                &mut self.state,
4814                cache_dir.as_deref(),
4815                "gate",
4816                "gate",
4817                &winner.worktree,
4818                &head,
4819                timeout,
4820                "final gate",
4821                |state, budget| {
4822                    let shell = shell.clone();
4823                    let gate_commands = gate_commands.clone();
4824                    let worktree = winner.worktree.clone();
4825                    async move {
4826                        let (outcomes, timed_out_pids) = run_commands(
4827                            state,
4828                            "gate",
4829                            "gate",
4830                            0,
4831                            &shell,
4832                            &gate_commands,
4833                            &worktree,
4834                            budget,
4835                        )
4836                        .await;
4837                        (outcomes, false, timed_out_pids)
4838                    }
4839                },
4840            )
4841            .await;
4842            outcomes
4843        };
4844        if outcomes.is_empty() {
4845            // Nothing configured to check — distinct from every other
4846            // silence in this run's event log, since an empty `gate` alone
4847            // no longer says whether the gate ran at all (see
4848            // `RunState::gate_ran`'s own doc).
4849            self.state.event(
4850                "gate",
4851                "no gate commands configured; nothing to check, passing",
4852            );
4853        }
4854        for o in &outcomes {
4855            self.state.event(
4856                "gate",
4857                format!(
4858                    "`{}` -> {}",
4859                    o.command,
4860                    if o.ok() {
4861                        "pass".to_owned()
4862                    } else {
4863                        format!(
4864                            "FAIL ({:?})\n{}",
4865                            o.code,
4866                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
4867                        )
4868                    }
4869                ),
4870            );
4871        }
4872        Ok(outcomes)
4873    }
4874
4875    /// One bounded fix round for a failing gate.
4876    ///
4877    /// The fixer is told the failure came from the gate itself, not from a
4878    /// reviewer, and is shown the failed commands, their exit codes and a tail
4879    /// of their output - whatever `[verify].gate` holds, nothing here knows
4880    /// what those commands run. Only a normal non-zero exit that printed
4881    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
4882    /// command or a full disk says nothing about the code, and a fixer sent
4883    /// after it can only appease the machine. The round is judged by what git
4884    /// says moved, never by the fixer's own report, and `verify.e2e` runs
4885    /// again before the gate does, so a fix cannot trade a green gate for a
4886    /// red e2e unnoticed.
4887    async fn gate_fix_round(
4888        &mut self,
4889        winner: &Candidate,
4890        outcomes: &[CommandOutcome],
4891    ) -> Result<GateFix> {
4892        let cap = self.state.config.graph.gate_fix_rounds;
4893        let spent = self.state.gate_fixes.len();
4894        if spent >= cap {
4895            if cap > 0 {
4896                self.state.event(
4897                    "gate",
4898                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
4899                );
4900            }
4901            return Ok(GateFix::Stop);
4902        }
4903        if !gate_fixable(outcomes) {
4904            self.state.event(
4905                "gate",
4906                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
4907                 command or similar); not spending a fix round on it",
4908            );
4909            return Ok(GateFix::Stop);
4910        }
4911        let min_free = self.state.config.disk.min_free_bytes;
4912        if min_free > 0 {
4913            match crate::disk::free_bytes(&winner.worktree) {
4914                Ok(free) if crate::disk::enough_space(free, min_free) => {}
4915                Ok(free) => {
4916                    self.state.event(
4917                        "gate",
4918                        format!(
4919                            "only {free} bytes free ({min_free} required by `[disk] \
4920                             min_free_bytes`); not spending a fix round on a failure the disk \
4921                             may explain"
4922                        ),
4923                    );
4924                    return Ok(GateFix::Stop);
4925                }
4926                Err(e) => {
4927                    self.state.event(
4928                        "gate",
4929                        format!("free disk space could not be measured ({e:#}); no fix round"),
4930                    );
4931                    return Ok(GateFix::Stop);
4932                }
4933            }
4934        }
4935
4936        let attempt = spent + 1;
4937        let run_id = self.state.id.clone();
4938        let prompts = self.state.config.prompts.clone();
4939        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
4940        let base = self.landing_base();
4941        let (fix_spec, fix_seat_key) = self.fixer_spec(winner);
4942        let seat = self.seat(&fix_seat_key, &fix_spec.id);
4943        let job = SeatJob {
4944            prompt: prompt::gate_fix(
4945                &self.state.instruction,
4946                &failed,
4947                attempt,
4948                cap,
4949                &self.state.config.graph.language,
4950            ),
4951            spec: fix_spec,
4952            seat,
4953            cwd: winner.worktree.clone(),
4954            timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
4955            allow_write: true,
4956            sessions: self.state.config.graph.sessions,
4957            artifacts: agent::artifacts_dir(&self.state.dir()),
4958            stem: format!("gate-fix-{attempt}"),
4959        };
4960        self.state.event(
4961            "gate",
4962            format!("gate failed; gate-fix round {attempt} of {cap}"),
4963        );
4964        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
4965        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
4966        let cache = self.state.config.cache_dir();
4967        let ctx = WaveCtx {
4968            run: &run_id,
4969            node: "gate-fix",
4970            prompts: &prompts,
4971            cache: cache.as_deref(),
4972            round: None,
4973        };
4974        let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4975        let mut record = GateFixRecord {
4976            agent: seat.agent.clone(),
4977            failed,
4978            notes: String::new(),
4979            committed: false,
4980            error: None,
4981        };
4982        match out {
4983            AgentOutcome::Ok(o) => {
4984                // A missing report is not a failed fix: the round is judged
4985                // by the tree below, and the report only carries prose.
4986                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
4987                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
4988                }
4989            }
4990            AgentOutcome::Dropped(_) => {
4991                record.error = Some("the CLI dropped the stream".to_owned());
4992            }
4993            AgentOutcome::Quota(o) => {
4994                self.state.quota.push(QuotaLoss {
4995                    seat: seat.key.clone(),
4996                    node: "gate-fix".to_owned(),
4997                    at: Timestamp::now(),
4998                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4999                });
5000                record.error = Some("rate limited (quota); fixer could not run".to_owned());
5001            }
5002            AgentOutcome::Failed(e) => record.error = Some(e),
5003        }
5004        self.state.seats.insert(seat.key.clone(), seat);
5005        if let Ok(r) = git::rescue_commit(
5006            &winner.worktree,
5007            &format!("magi: gate fix {attempt} (uncommitted work)"),
5008        )
5009        .await
5010        {
5011            self.state.note_withheld("gate-fix", &r.withheld);
5012        }
5013        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5014        record.committed = after != before;
5015        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
5016        let note = record.error.clone();
5017        self.state.gate_fixes.push(record);
5018        self.state.save()?;
5019        if !changed {
5020            self.state.event(
5021                "gate",
5022                match note {
5023                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
5024                    None => format!("gate-fix round {attempt}: the tree did not change"),
5025                },
5026            );
5027            return Ok(GateFix::Stop);
5028        }
5029        self.state.event(
5030            "gate",
5031            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
5032        );
5033
5034        let commands = self.state.config.verify.e2e.clone();
5035        if !commands.is_empty() {
5036            let shell = self.state.config.shell();
5037            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5038            let cache_dir = self.state.config.cache_dir();
5039            let context = format!("gate-fix round {attempt}");
5040            let (e2e, _) = with_cache_lease(
5041                &mut self.state,
5042                cache_dir.as_deref(),
5043                "e2e",
5044                "e2e",
5045                &winner.worktree,
5046                &after,
5047                timeout,
5048                &context,
5049                |state, budget| {
5050                    let shell = shell.clone();
5051                    let commands = commands.clone();
5052                    let context = context.clone();
5053                    let worktree = winner.worktree.clone();
5054                    async move {
5055                        run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
5056                            .await
5057                    }
5058                },
5059            )
5060            .await;
5061            if verify_inconclusive(&e2e) {
5062                return Ok(GateFix::Defer);
5063            }
5064            if e2e.iter().any(|o| !o.ok()) {
5065                self.state.event(
5066                    "gate",
5067                    format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
5068                );
5069                return Ok(GateFix::Stop);
5070            }
5071        }
5072        Ok(GateFix::Retry)
5073    }
5074
5075    // ---------------------------------------------------------------- merge
5076
5077    async fn merge(&mut self) -> Result<()> {
5078        // Same reasoning as `gate`: ask the review and gate records directly
5079        // rather than `status`, which a solo-candidate `judge`/`deliberate`
5080        // skip can rewrite on reentry to something that no longer says
5081        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
5082        // so a hand-off (open findings, green verification) reaches merge
5083        // exactly like a genuinely clean round does.
5084        //
5085        // A run resumed mid-`land` never reaches here at all: `execute`
5086        // recognises `RunStatus::Landing` before it even calls `prep`, and
5087        // routes straight to `run_land` instead. That has to happen a level
5088        // up from this function, not with a check in here, because
5089        // `review_loop`'s own status recomputation (see its doc) runs
5090        // *before* `merge` on every reentry and would otherwise overwrite
5091        // the `Landing` marker with `Gating` before this node ever saw it.
5092        if self
5093            .state
5094            .base_sync
5095            .as_ref()
5096            .is_some_and(|s| s.conflict.is_some())
5097            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5098                != Some(RunStatus::Gating)
5099            // `gate_ran == false` is not "passed" - `gate` leaves it false
5100            // both before it has ever run and when its last attempt was
5101            // resource-blocked (see `Runner::gate`'s own doc), and neither is
5102            // permission to merge on nothing but the review record. Only a
5103            // gate that actually ran - zero commands configured and
5104            // vacuously passed, or one or more that all exited 0 - may
5105            // proceed; `RunState::gate_status` is the single place that
5106            // reading is computed.
5107            || !self.state.gate_status().ok()
5108        {
5109            return Ok(());
5110        }
5111        // This node's own record, not `status`: `status == Ready` is not
5112        // unique to the harmless `MergeMode::None` path this line was
5113        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
5114        // run's PR was closed without merging — and on that run `mode` is
5115        // still `Pr`, so a reentry that fell through here would push and
5116        // open a second pull request. `self.state.merge` is set exactly once
5117        // this node (or `land`) has already produced a verdict, under every
5118        // mode, which is what "already done" actually means here.
5119        if self.state.merge.is_some() {
5120            return Ok(());
5121        }
5122        let Some(winner) = self.state.winner().cloned() else {
5123            return Ok(());
5124        };
5125        let repo = self.state.repo.clone();
5126        let base = self.state.base_branch.clone();
5127        let mode = self.state.config.merge.mode;
5128        let style = self.state.config.merge.style;
5129        let pr = pr_message(&self.state, winner.label);
5130        let message = pr.commit_message();
5131
5132        let outcome = match mode {
5133            MergeMode::None => MergeOutcome {
5134                mode,
5135                ok: true,
5136                detail: manual_merge_command(style, &repo, &winner.branch, &message),
5137            },
5138            MergeMode::Local => {
5139                let on = git::current_branch(&repo).await?;
5140                if on.as_deref() != Some(base.as_str()) {
5141                    MergeOutcome {
5142                        mode,
5143                        ok: false,
5144                        detail: format!(
5145                            "{} has {} checked out, not the base branch {base}",
5146                            repo.display(),
5147                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
5148                        ),
5149                    }
5150                } else if !git::is_clean(&repo).await? {
5151                    MergeOutcome {
5152                        mode,
5153                        ok: false,
5154                        detail: format!("{} is dirty; refusing to merge", repo.display()),
5155                    }
5156                } else {
5157                    let out = match style {
5158                        MergeStyle::Merge => {
5159                            git::merge_no_ff(&repo, &winner.branch, &message).await?
5160                        }
5161                        MergeStyle::Squash => {
5162                            git::merge_squash(&repo, &winner.branch, &message).await?
5163                        }
5164                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
5165                    };
5166                    MergeOutcome {
5167                        mode,
5168                        ok: out.ok(),
5169                        detail: if out.ok() { out.stdout } else { out.stderr },
5170                    }
5171                }
5172            }
5173            MergeMode::Pr => {
5174                let remote = self.state.config.merge.remote.clone();
5175                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
5176                if !pushed.ok() {
5177                    MergeOutcome {
5178                        mode,
5179                        ok: false,
5180                        detail: pushed.stderr,
5181                    }
5182                } else {
5183                    let out =
5184                        gh_pr_create(&winner.worktree, &base, &winner.branch, &pr.title, &pr.body)
5185                            .await;
5186                    match out {
5187                        Ok(url) => MergeOutcome {
5188                            mode,
5189                            ok: true,
5190                            detail: url,
5191                        },
5192                        Err(e) => MergeOutcome {
5193                            mode,
5194                            ok: false,
5195                            detail: e.to_string(),
5196                        },
5197                    }
5198                }
5199            }
5200        };
5201
5202        self.state.status = match (mode, outcome.ok) {
5203            (MergeMode::None, _) => RunStatus::Ready,
5204            (_, true) => RunStatus::Merged,
5205            (_, false) => RunStatus::Blocked,
5206        };
5207        self.state.event(
5208            "merge",
5209            format!(
5210                "{:?}: {}",
5211                mode,
5212                outcome.detail.lines().next().unwrap_or("")
5213            ),
5214        );
5215        self.state.merge = Some(outcome);
5216        self.state.save()?;
5217
5218        // The PR is open and the run would historically stop here, leaving the
5219        // operator to watch checks, feed review comments back to a fixer, and
5220        // merge. That was done by hand six times in one session before this
5221        // existed. Opt-in, because merging is the one irreversible thing magi
5222        // can do to a repository.
5223        if self.state.config.graph.land
5224            && mode == MergeMode::Pr
5225            && self.state.status == RunStatus::Merged
5226        {
5227            self.run_land().await?;
5228        }
5229        // `run_land` may have left `status` at `Landing` - still waiting on
5230        // CI or the owner's approval, not actually settled - so this has to
5231        // read whatever `status` ended up as here, not the `Merged` this
5232        // function set a few lines up.
5233        self.settle_questions();
5234        Ok(())
5235    }
5236
5237    /// Enter `land`.
5238    ///
5239    /// Shared between a fresh run's first pass through [`Runner::merge`] and
5240    /// a resumed run's re-entry. `land::land` itself is what serialises the
5241    /// two git-mutating moments inside the loop — the rebase push and
5242    /// `gh pr merge` — per repository (see its own doc); nothing here needs
5243    /// to hold a lock across the whole call, and doing so would serialise
5244    /// this run's CI wait against a *different* run's land-approval resume
5245    /// in the same repository, which is exactly the "must not wait on
5246    /// another task" property the daemon's slot-freeing exists to give.
5247    async fn run_land(&mut self) -> Result<()> {
5248        let url = self
5249            .state
5250            .merge
5251            .as_ref()
5252            .map(|m| m.detail.clone())
5253            .unwrap_or_default();
5254        let url = url.lines().next().unwrap_or("").trim().to_owned();
5255        if !url.starts_with("http") {
5256            return Ok(());
5257        }
5258        // A land failure is not a lost run: the work is on a branch and the
5259        // pull request is open, which is exactly where a human takes over.
5260        match land::land(&mut self.state, &url).await {
5261            Ok(pr) if self.state.parked => {
5262                // `land` already saved the parked marker; nothing here
5263                // overrides `status` back to a terminal value while an
5264                // approval is still outstanding.
5265                let _ = pr;
5266            }
5267            Ok(pr) => {
5268                self.state.status = match pr.state {
5269                    land::PrLifecycle::Merged => RunStatus::Merged,
5270                    _ => RunStatus::Blocked,
5271                };
5272                // Downstream of a confirmed merge only - see
5273                // `bump::should_release_bump`'s own doc for why this one
5274                // check covers all three of `land`'s success paths.
5275                // Best-effort: the run already landed, so a failure here
5276                // (the decision call, `gh`, `cargo`) is recorded and never
5277                // turns a landed run into a failed one.
5278                if bump::should_release_bump(self.state.status)
5279                    && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
5280                {
5281                    // Deliberately only an event: most `Err`s here mean the
5282                    // bump did not apply (no `Cargo.toml`, no agent
5283                    // installed, an unusable decision), not that a release
5284                    // PR is stranded. `after_merge` raises its own notice
5285                    // once a PR exists and needs a human.
5286                    self.state
5287                        .event("bump", format!("release bump skipped: {e:#}"));
5288                }
5289                self.state.save()?;
5290            }
5291            Err(e) => {
5292                self.state.status = RunStatus::Blocked;
5293                self.state.event("land", format!("gave up: {e}"));
5294                self.state.save()?;
5295            }
5296        }
5297        Ok(())
5298    }
5299
5300    // -------------------------------------------------------------- helpers
5301
5302    /// Fetch or create a seat, keeping its conversation across nodes.
5303    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
5304        if let Some(existing) = self.state.seats.get(key)
5305            && existing.agent == agent
5306        {
5307            return existing.clone();
5308        }
5309        let fresh = SeatState::new(key, agent, self.state.seed);
5310        self.state.seats.insert(key.to_owned(), fresh.clone());
5311        fresh
5312    }
5313
5314    /// A candidate rendered for judging, with the leak policy applied.
5315    fn view(&self, c: &Candidate) -> CandidateView {
5316        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
5317            .unwrap_or_default();
5318        let (patch, _) = blind::sanitize_patch(
5319            &format!("candidate {} patch", c.label),
5320            &raw,
5321            &self.state.config.blind,
5322        );
5323        CandidateView {
5324            label: c.label,
5325            branch: c.branch.clone(),
5326            summary: c.summary.clone(),
5327            stat: c.stat.clone(),
5328            patch,
5329        }
5330    }
5331
5332    /// The full candidate set as prompt text, for seats with no live session.
5333    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
5334        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
5335        prompt::judge(
5336            "(see above)",
5337            &views,
5338            self.roles.judges.len(),
5339            base_short,
5340            "en",
5341        )
5342    }
5343
5344    /// Anonymised transcript for judge `self_idx`.
5345    ///
5346    /// The initial rankings are always the opening statements. Seeding them
5347    /// only when no turn had been taken yet meant every judge after the first
5348    /// argued against a single voice instead of against the actual split — the
5349    /// disagreement is the information, so it is always on the table.
5350    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
5351        let mut turns = Vec::new();
5352        for j in &self.state.judgements {
5353            if j.ranking.is_empty() {
5354                continue;
5355            }
5356            let reasons = j
5357                .reasons
5358                .iter()
5359                .map(|(k, v)| format!("- {k}: {v}"))
5360                .collect::<Vec<_>>()
5361                .join("\n");
5362            turns.push(Turn {
5363                who: format!("Judge {} (opening ranking)", j.judge),
5364                is_self: j.judge == self_idx + 1,
5365                body: format!(
5366                    "Ranked {}{}{reasons}",
5367                    j.ranking.iter().collect::<String>(),
5368                    if reasons.is_empty() {
5369                        ""
5370                    } else {
5371                        ", because:\n"
5372                    }
5373                ),
5374            });
5375        }
5376        for t in self
5377            .state
5378            .deliberation
5379            .iter()
5380            .flat_map(|r| r.turns.iter())
5381            .chain(current)
5382        {
5383            turns.push(Turn {
5384                who: format!("Judge {}", t.judge),
5385                is_self: t.judge == self_idx + 1,
5386                body: t.body.clone(),
5387            });
5388        }
5389        turns
5390    }
5391}
5392
5393/// Does this seat still hold the context a follow-up prompt would rely on?
5394fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
5395    agent::has_session(spec.kind, seat, sessions)
5396}
5397
5398/// The next entry in `roster` after `start`, never wrapping back to the
5399/// front, whose id is not in `tried` yet.
5400///
5401/// Starts one past `start` rather than at the front of `roster`: `start` is
5402/// the seat's own original position, and a seat whose candidate slot already
5403/// sits on the roster's second entry must fall through to the third next, not
5404/// restart at the first — which is very likely a different candidate's own
5405/// agent already. Never wraps back past `start`, for the same reason: an
5406/// entry earlier in the roster than the seat's own position is almost
5407/// certainly some *other* candidate slot's own agent, and once the tail of
5408/// the roster is exhausted there are no more untried agents for *this* seat
5409/// to fall through to — the caller's fallback chain ends there, exactly as
5410/// "no further untried agents remain in the list for that seat" asks for.
5411///
5412/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
5413/// the same id twice (an operator's `roles.implementers` typo, or a
5414/// `[[agents]]` list reused across roles) must not let
5415/// [`Runner::resume_quota_losses`] retry that id forever — one forward pass
5416/// over `roster` either finds an untried id or runs out, so this always
5417/// terminates regardless of duplicates.
5418fn next_untried_implementer<'a>(
5419    roster: &'a [AgentSpec],
5420    start: usize,
5421    tried: &BTreeSet<String>,
5422) -> Option<&'a AgentSpec> {
5423    roster
5424        .get(start + 1..)?
5425        .iter()
5426        .find(|s| !tried.contains(&s.id))
5427}
5428
5429/// Did this reply report running a command whose own CLI never confirmed an
5430/// exit status?
5431///
5432/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
5433/// command *finished* (see that type's own doc), so this can only be `true`
5434/// for a command whose completion event carried no readable exit code — not
5435/// for one that simply is not mentioned at all. That is the one signal this
5436/// crate can read, from the same record `state.jobs` renders, about a reply
5437/// standing next to work its own CLI cannot vouch for finishing; it is
5438/// deliberately not a check on the exit code's *value* (a fixer legitimately
5439/// runs a command that fails mid-iteration before it succeeds) and not a
5440/// guess at a command still running in the background (which emits no event
5441/// at all, and so leaves no evidence here to find).
5442fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
5443    commands.iter().any(|c| c.exit_code.is_none())
5444}
5445
5446/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
5447/// trusted as a verified no-op — the adoption guard's own text-level half.
5448///
5449/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
5450/// timed out): a marker only earns the benefit of the doubt from a turn the
5451/// CLI itself vouches for finishing properly, the same house style
5452/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
5453/// *fix* report to for `commands`. A candidate that timed out, exited
5454/// non-zero, or left a command unconfirmed is read as the ordinary loss it
5455/// is, whatever prose it wrote — this returns `None` before it ever looks at
5456/// `text`. The remaining guards (the tree really is empty, the evidence is
5457/// non-empty) are the caller's: this only reads what the reply *claimed*.
5458fn verified_noop_claim(
5459    usable: bool,
5460    commands: &[agent::CommandEvidence],
5461    text: &str,
5462) -> Option<String> {
5463    (usable && !has_unconfirmed_command(commands))
5464        .then(|| verdict::verified_noop(text))
5465        .flatten()
5466}
5467
5468fn short(commit: &str) -> String {
5469    commit.chars().take(7).collect()
5470}
5471
5472fn make_executable(path: &Path) -> Result<()> {
5473    #[cfg(unix)]
5474    {
5475        use std::os::unix::fs::PermissionsExt as _;
5476        let mut perms = std::fs::metadata(path)?.permissions();
5477        perms.set_mode(0o755);
5478        std::fs::set_permissions(path, perms)?;
5479    }
5480    #[cfg(not(unix))]
5481    {
5482        let _ = path;
5483    }
5484    Ok(())
5485}
5486
5487/// What every seat in one batch shares: where the answers are attributed, the
5488/// prompt overlay they inherit, and the build cache they are told to use.
5489///
5490/// A struct rather than four more parameters: `wave` also needs the run's
5491/// state (to record who is answering right now) and the attempt number, and
5492/// eight positional arguments is both unreadable and a clippy error.
5493struct WaveCtx<'a> {
5494    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
5495    /// paid for it.
5496    run: &'a str,
5497    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
5498    node: &'a str,
5499    prompts: &'a Prompts,
5500    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
5501    cache: Option<&'a Path>,
5502    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
5503    /// `JobRecord::round`. `None` for every other node.
5504    round: Option<usize>,
5505}
5506
5507/// Run one job, honouring the parallelism budget.
5508async fn run_one(
5509    job: SeatJob,
5510    sem: Arc<Semaphore>,
5511    ctx: &WaveCtx<'_>,
5512    state: &mut RunState,
5513    attempt: usize,
5514) -> (SeatState, AgentOutcome) {
5515    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
5516        .await
5517        .pop()
5518        .expect("one job in, one result out");
5519    (seat, out)
5520}
5521
5522/// Run every job concurrently, capped by the semaphore, preserving order.
5523///
5524/// Every seat in the batch is recorded into [`RunState::active`] before the
5525/// wave starts and cleared as each answer lands, so the run's own record says
5526/// who is still being waited on rather than only who finished.
5527async fn wave(
5528    jobs: Vec<SeatJob>,
5529    sem: Arc<Semaphore>,
5530    ctx: &WaveCtx<'_>,
5531    state: &mut RunState,
5532    attempt: usize,
5533) -> Vec<(usize, SeatState, AgentOutcome)> {
5534    let WaveCtx {
5535        run,
5536        node,
5537        prompts,
5538        cache,
5539        round,
5540    } = *ctx;
5541    for job in &jobs {
5542        state.seat_started(node, &job.seat.key, job.timeout, attempt);
5543    }
5544    if let Err(e) = state.save() {
5545        // A failed persist of "who is answering right now" must not abort the
5546        // wave: the seats are already being asked, and the alternative is
5547        // losing the answers to save a status line nobody may even be
5548        // watching.
5549        tracing::warn!("could not persist in-progress seats: {e:#}");
5550    }
5551    // Hold the shared build cache's lease for the whole batch, not per job:
5552    // several candidates (an implement wave) or a fixer legitimately share
5553    // one cache concurrently within this run, and that stays untouched — a
5554    // single lease taken once for the whole wave and released once it is
5555    // done is what stops a *different* borrower (another run's own wave, its
5556    // e2e/gate, a human's `magi review`) from interleaving a build into the
5557    // same directory while this one is in flight. Best-effort, not
5558    // all-or-nothing: a wave that cannot get the lease within its own
5559    // longest job's budget still runs — an hour of paid implementer calls is
5560    // not thrown away over cache contention — but every write-allowed seat
5561    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
5562    // below), the same fallback a read-only seat always gets, rather than
5563    // building into a directory this run was never granted. The identity
5564    // record is still invalidated below either way, so the next tracked
5565    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
5566    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
5567    let wait_started = Instant::now();
5568    let cache_guard = if let Some(cache_dir) = cache {
5569        if jobs_had_a_writer {
5570            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
5571            let budget = jobs
5572                .iter()
5573                .map(|j| j.timeout)
5574                .max()
5575                .unwrap_or(Duration::from_secs(60));
5576            acquire_cache_lease(state, cache_dir, &owner, budget, node)
5577                .await
5578                .ok()
5579        } else {
5580            None
5581        }
5582    } else {
5583        None
5584    };
5585    // Carved out of each job's own budget, not added on top of it: a seat
5586    // that waited behind the lease must not also get its full timeout
5587    // afterward, or a run contended on the cache could double the time it
5588    // spends per wave. `saturating_sub` floors at zero rather than
5589    // wrapping - a job whose whole budget was spent waiting starts with
5590    // none left, which is the honest number, not a free minimum.
5591    let waited_for_lease = wait_started.elapsed();
5592    let mut set = tokio::task::JoinSet::new();
5593    let overlay = prompts.overlay(node);
5594    for (i, mut job) in jobs.into_iter().enumerate() {
5595        job.timeout = job.timeout.saturating_sub(waited_for_lease);
5596        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
5597        if cache.is_some() {
5598            job.prompt.push('\n');
5599            job.prompt
5600                .push_str(&prompt::build_cache_note(node, job.allow_write));
5601        }
5602        let sem = Arc::clone(&sem);
5603        let run = run.to_owned();
5604        let node = node.to_owned();
5605        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
5606        // `prompt::build_cache_note`'s doc for why setting it anyway is
5607        // exactly how a sandboxed reviewer's write refusal got reported as a
5608        // defect in the patch, not a property of its own seat. And a
5609        // write-allowed one is handed it only when the lease above was
5610        // actually acquired: a wave that could not get it (`cache_guard` is
5611        // `None`, see its own comment) must not send seats to build into a
5612        // directory this run does not hold - that is the exact concurrent,
5613        // unmanaged-write race this module exists to prevent, not something
5614        // "proceeding anyway" is allowed to reintroduce.
5615        let cache = cache
5616            .filter(|_| job.allow_write && cache_guard.is_some())
5617            .map(Path::to_path_buf);
5618        set.spawn(async move {
5619            let _permit = sem.acquire().await;
5620            let mut seat = job.seat;
5621            let out = agent::invoke(
5622                &job.spec,
5623                &mut seat,
5624                &Invocation {
5625                    cwd: &job.cwd,
5626                    prompt: &job.prompt,
5627                    timeout: job.timeout,
5628                    allow_write: job.allow_write,
5629                    sessions: job.sessions,
5630                    artifacts: &job.artifacts,
5631                    stem: &job.stem,
5632                    run: &run,
5633                    node: &node,
5634                    cache_dir: cache.as_deref(),
5635                    attachments: &[],
5636                },
5637            )
5638            .await;
5639            let out = match out {
5640                Ok(o) if o.usable() => AgentOutcome::Ok(o),
5641                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
5642                // Billed work the CLI failed to hand over is not an ordinary
5643                // failure, but its text is the CLI's raw error JSON, not an
5644                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
5645                // read it as one by forgetting to check. `usable()` is always
5646                // false here (dropped implies an empty response), so this has
5647                // to be checked before the catch-all `Failed` below or the
5648                // one shape this exists for is lost with the rest.
5649                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
5650                Ok(o) if o.timed_out => AgentOutcome::Failed("timed out".to_owned()),
5651                Ok(o) => AgentOutcome::Failed(format!(
5652                    "exited with {:?} and no usable output",
5653                    o.exit_code
5654                )),
5655                Err(e) => AgentOutcome::Failed(e.to_string()),
5656            };
5657            (i, seat, out)
5658        });
5659    }
5660    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
5661    while let Some(joined) = set.join_next().await {
5662        let (i, seat, out) = match joined {
5663            Ok(v) => v,
5664            // No seat to clear: a panicked task never reported which one it
5665            // was. The defensive sweep below this loop is what stops that
5666            // seat's `active` entry from surviving forever.
5667            Err(e) => {
5668                tracing::error!("agent task panicked: {e}");
5669                continue;
5670            }
5671        };
5672        state.seat_finished(&seat.key);
5673        record_jobs(state, node, round, &seat.key, &out);
5674        if let Err(e) = state.save() {
5675            tracing::warn!("could not persist a seat's completion: {e:#}");
5676        }
5677        if collected.len() <= i {
5678            collected.resize_with(i + 1, || None);
5679        }
5680        collected[i] = Some((i, seat, out));
5681    }
5682    // Belt-and-braces for the panic branch above: every seat this exact batch
5683    // started shares this `(node, attempt)` pair, and every seat that finished
5684    // normally already cleared itself, so anything left tagged with it here
5685    // can only be a panicked task's leftover. Cleared unconditionally rather
5686    // than left to read as still answering forever.
5687    if state
5688        .active
5689        .values()
5690        .any(|a| a.node == node && a.attempt == attempt)
5691    {
5692        state
5693            .active
5694            .retain(|_, a| !(a.node == node && a.attempt == attempt));
5695        if let Err(e) = state.save() {
5696            tracing::warn!("could not persist the end of a wave: {e:#}");
5697        }
5698    }
5699    // Whether or not the lease above was actually held, several worktrees
5700    // may just have built into the cache with nothing here able to name one
5701    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
5702    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
5703    // clean it might not have strictly needed; trusting a stale match would
5704    // cost it a wrong answer.
5705    if let Some(cache_dir) = cache
5706        && jobs_had_a_writer
5707    {
5708        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
5709    }
5710    if let Some(guard) = cache_guard {
5711        guard.release();
5712    }
5713    collected.into_iter().flatten().collect()
5714}
5715
5716/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
5717/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
5718/// this is data collection, not the fix-specific completion contract in
5719/// [`Runner::continue_fix_report`], and applies regardless of which node
5720/// asked.
5721///
5722/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
5723/// evidence from; `Failed` does not, and correctly contributes nothing — a
5724/// timeout or crash is not itself evidence about a command the seat may have
5725/// started.
5726fn record_jobs(
5727    state: &mut RunState,
5728    node: &str,
5729    round: Option<usize>,
5730    seat: &str,
5731    out: &AgentOutcome,
5732) {
5733    let commands: &[agent::CommandEvidence] = match out {
5734        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
5735        AgentOutcome::Failed(_) => &[],
5736    };
5737    let checked_at = Timestamp::now();
5738    for c in commands {
5739        state.jobs.push(JobRecord {
5740            node: node.to_owned(),
5741            round,
5742            seat: seat.to_owned(),
5743            id: c.id.clone(),
5744            description: c.description.clone(),
5745            checked_at,
5746            status: match c.exit_code {
5747                Some(0) => JobStatus::Completed,
5748                Some(_) => JobStatus::Failed,
5749                None => JobStatus::Unknown,
5750            },
5751            exit_code: c.exit_code,
5752            result_summary: c.result_summary.clone(),
5753            source: c.source.clone(),
5754        });
5755    }
5756}
5757
5758/// Is a review round clean, given how many reviewer seats answered against
5759/// how many the round expected?
5760///
5761/// A seat that never answered (timeout, crash, unparsable output) is not a
5762/// seat that read the patch and found nothing — treating it as such is
5763/// exactly the bug this function exists to close. Under the default `block`
5764/// policy a missing seat can never be clean; `warn` still requires the seats
5765/// that *did* answer to have found nothing blocking and verification to be
5766/// green.
5767///
5768/// `quota_missing` narrows that `block` default for exactly one cause of
5769/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
5770/// a session limit lifts by the very next round buys nothing — the seat is
5771/// asked again with the same quota — so once every missing seat is accounted
5772/// for by a quota loss (and at least one seat *did* answer, so a decision has
5773/// something to rest on) the round is decided on the panel that could answer,
5774/// same as `warn` would. A panel that lost every seat to quota is not
5775/// decided here: `answered == 0` falls through to the existing `block`
5776/// fallback so a fully collapsed panel still waits rather than landing on no
5777/// review at all.
5778fn round_is_clean(
5779    blocking: usize,
5780    e2e_ok: bool,
5781    answered: usize,
5782    expected: usize,
5783    quota_missing: usize,
5784    policy: IncompleteReviewPolicy,
5785) -> bool {
5786    if blocking != 0 || !e2e_ok {
5787        return false;
5788    }
5789    if answered == expected || policy == IncompleteReviewPolicy::Warn {
5790        return true;
5791    }
5792    answered > 0 && expected - answered <= quota_missing
5793}
5794
5795/// The review loop's own conclusion, derived entirely from its persisted
5796/// round records and the round budget that produced them — never from
5797/// `status`, so a reentry (or `gate`/`merge` reading it independently)
5798/// recomputes the identical answer regardless of what an earlier node in the
5799/// same walk, or a previous walk, did to `status`.
5800///
5801/// `None` while more rounds remain to try, including when review never ran
5802/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
5803/// gone clean, or the budget is spent, or the tree has stopped moving (see
5804/// [`STAGNANT_LIMIT`]), the answer is one of two things:
5805///
5806/// - An incomplete panel that raised nothing is missing input, not a
5807///   verified tree — never a hand-off candidate, whatever verification said
5808///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
5809/// - Otherwise, green e2e on the last round hands off (see
5810///   [`Runner::stop_reviewing`]); red e2e blocks.
5811///
5812/// A last round whose own verification is still `ResourceBlocked` — magi
5813/// itself never got a command to run, not evidence the patch is broken —
5814/// is neither: this returns `None` for it too, the same as "more rounds
5815/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
5816/// handling of that shape) instead of this cheap recomputation guessing a
5817/// verdict a real attempt never produced.
5818fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
5819    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
5820        return Some(RunStatus::Gating);
5821    }
5822    let last = reviews.last()?;
5823    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
5824    if reviews.len() < max_rounds && !stagnant {
5825        return None;
5826    }
5827    if last.incomplete() && last.blocking == 0 {
5828        return Some(RunStatus::Blocked);
5829    }
5830    if last.e2e_status() == E2eStatus::ResourceBlocked {
5831        return None;
5832    }
5833    Some(if last.e2e.iter().all(CommandOutcome::ok) {
5834        RunStatus::Gating
5835    } else {
5836        RunStatus::Blocked
5837    })
5838}
5839
5840/// How long a re-ask may take, given the budget the first attempt had.
5841///
5842/// A `nudged` retry is a request to restate an answer the seat has already
5843/// worked out: it carries no new work, so it does not deserve the original
5844/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
5845/// 133 seconds while a third sat for over ten minutes on a resumed session
5846/// holding 410 KB of prior output - and because the retry had inherited the
5847/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
5848/// judging round whose other seats were long finished.
5849///
5850/// A quarter of the budget, with a floor so that a deliberately short timeout
5851/// does not collapse to nothing. A retry that re-sends the whole prompt
5852/// (because the seat kept no context) is the original job again, and keeps the
5853/// original budget.
5854fn retry_budget(full: Duration, nudged: bool) -> Duration {
5855    if nudged {
5856        (full / 4).max(Duration::from_secs(120)).min(full)
5857    } else {
5858        full
5859    }
5860}
5861
5862/// Run a wave and parse each reply, re-asking the seats whose reply was
5863/// unusable.
5864///
5865/// The re-ask is a nudge rather than the whole prompt again when the seat still
5866/// holds its conversation, which is the difference between a cheap retry and
5867/// paying for the entire candidate set twice.
5868///
5869/// A seat that hits a rate limit is **not** re-asked: the same call will fail
5870/// the same way until the limit resets, so spending a retry attempt on it is
5871/// pure waste. Its loss is recorded in `losses` and it is returned as a failure
5872/// like any other absent seat — the caller decides whether the panel still has
5873/// a quorum.
5874#[allow(clippy::too_many_arguments)]
5875async fn ask_json_wave<T>(
5876    jobs: Vec<SeatJob>,
5877    sem: Arc<Semaphore>,
5878    retries: usize,
5879    ctx: &WaveCtx<'_>,
5880    losses: &mut Vec<QuotaLoss>,
5881    state: &mut RunState,
5882    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
5883) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
5884where
5885    T: serde::de::DeserializeOwned + Send + 'static,
5886{
5887    let n = jobs.len();
5888    let originals: Vec<SeatJob> = jobs;
5889    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
5890    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
5891    // Which attempt each seat's `done[i]` reflects — 0 for a first-ask
5892    // answer, N once it has gone through N nudges. Read back once this
5893    // returns, so a caller building a history record (`ReviewRecord`) can
5894    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
5895    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
5896    // field's own doc.
5897    let mut attempts_used: Vec<usize> = vec![0; n];
5898    let mut pending: Vec<usize> = (0..n).collect();
5899
5900    for attempt in 0..=retries {
5901        if pending.is_empty() {
5902            break;
5903        }
5904        let mut batch = Vec::with_capacity(pending.len());
5905        for &i in &pending {
5906            let src = &originals[i];
5907            // The prompt and the budget are one decision: a nudge restates
5908            // finished work, a re-sent prompt redoes it.
5909            let (prompt, timeout) = if attempt == 0 {
5910                (src.prompt.clone(), src.timeout)
5911            } else {
5912                let why = done[i]
5913                    .as_ref()
5914                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
5915                    .unwrap_or_else(|| "no parsable answer".to_owned());
5916                let nudge = prompt::nudge(&why);
5917                let nudged = has_context(&src.spec, &seats[i], src.sessions);
5918                let prompt = if nudged {
5919                    nudge
5920                } else {
5921                    format!("{}\n\n---\n\n{}", src.prompt, nudge)
5922                };
5923                (prompt, retry_budget(src.timeout, nudged))
5924            };
5925            batch.push(SeatJob {
5926                spec: src.spec.clone(),
5927                seat: seats[i].clone(),
5928                cwd: src.cwd.clone(),
5929                prompt,
5930                timeout,
5931                allow_write: src.allow_write,
5932                sessions: src.sessions,
5933                artifacts: src.artifacts.clone(),
5934                stem: if attempt == 0 {
5935                    src.stem.clone()
5936                } else {
5937                    format!("{}-retry{attempt}", src.stem)
5938                },
5939            });
5940        }
5941
5942        if attempt > 0 {
5943            let seats_out: Vec<&str> = pending
5944                .iter()
5945                .map(|&i| originals[i].seat.key.as_str())
5946                .collect();
5947            state.event(
5948                ctx.node,
5949                format!("retry {attempt}: re-asking {}", seats_out.join(", ")),
5950            );
5951        }
5952        let results = wave(batch, Arc::clone(&sem), ctx, state, attempt).await;
5953        let mut still = Vec::new();
5954        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
5955            seats[i] = seat;
5956            let (parsed, quota) = match out {
5957                AgentOutcome::Ok(o) => (
5958                    match verdict::extract_json::<T>(&o.text) {
5959                        Ok(v) => match validate(&v) {
5960                            Ok(()) => Ok((v, o)),
5961                            Err(e) => Err(e),
5962                        },
5963                        Err(e) => Err(e),
5964                    },
5965                    false,
5966                ),
5967                AgentOutcome::Quota(o) => {
5968                    losses.push(QuotaLoss {
5969                        seat: originals[i].seat.key.clone(),
5970                        node: ctx.node.to_owned(),
5971                        at: Timestamp::now(),
5972                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5973                    });
5974                    (
5975                        Err(anyhow::anyhow!("rate limited (quota); not retrying now")),
5976                        true,
5977                    )
5978                }
5979                // Not a parseable answer, but also not worth a special-cased
5980                // retry here: the nudge loop above already re-asks anything
5981                // that fails to parse, which is exactly what a dropped stream
5982                // needs. Just don't hand its raw error JSON to `extract_json`.
5983                AgentOutcome::Dropped(o) => {
5984                    let why = o
5985                        .dropped
5986                        .as_ref()
5987                        .map(|d| d.why.as_str())
5988                        .unwrap_or("the CLI ended the stream without delivering its answer");
5989                    (
5990                        Err(anyhow::anyhow!("the CLI dropped the stream ({why})")),
5991                        false,
5992                    )
5993                }
5994                AgentOutcome::Failed(e) => (Err(anyhow::anyhow!(e)), false),
5995            };
5996            let failed = parsed.is_err();
5997            done[i] = Some(parsed);
5998            attempts_used[i] = attempt;
5999            // Do not re-ask a rate-limited seat (quota) — a retry is known to
6000            // fail the same way; and never re-ask a seat that already parsed.
6001            if failed && !quota {
6002                still.push(i);
6003            }
6004        }
6005        pending = still;
6006    }
6007
6008    seats
6009        .into_iter()
6010        .zip(done)
6011        .zip(attempts_used)
6012        .map(|((seat, res), attempts)| {
6013            (
6014                seat,
6015                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
6016                attempts,
6017            )
6018        })
6019        .collect()
6020}
6021
6022/// Acquire the shared build cache's lease, waiting out contention within
6023/// `budget` (never past it — see AGENTS.md's build-cache section on why an
6024/// unbounded wait is never acceptable).
6025///
6026/// A first, non-blocking check happens before ever waiting; if it finds the
6027/// lease busy, that fact is logged as a `verify` event *and* flushed with
6028/// [`RunState::save`] immediately — not only once the wait finally succeeds
6029/// or gives up — so a `magi show` run by a different process while this one
6030/// is still waiting reads a `run.json` that says so, rather than whatever it
6031/// looked like before the wait started. The same applies to the terminal
6032/// failure: logged and saved before this returns `Err`, so a caller that
6033/// could not get the lease at all still leaves a legible record of why.
6034async fn acquire_cache_lease(
6035    state: &mut RunState,
6036    cache_dir: &Path,
6037    owner: &crate::cache::Owner,
6038    budget: Duration,
6039    context: &str,
6040) -> Result<crate::cache::Guard> {
6041    let home = crate::run::home();
6042    let started = Instant::now();
6043    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
6044        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(g),
6045        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
6046        Err(e) => {
6047            state.event(
6048                "verify",
6049                format!("{context}: could not check the shared build cache: {e:#}"),
6050            );
6051            if let Err(e2) = state.save() {
6052                tracing::warn!("could not persist a cache-check failure: {e2:#}");
6053            }
6054            return Err(e);
6055        }
6056    };
6057    state.event(
6058        "verify",
6059        format!(
6060            "{context}: waiting for the shared build cache at {} ({})",
6061            cache_dir.display(),
6062            busy.describe()
6063        ),
6064    );
6065    if let Err(e) = state.save() {
6066        tracing::warn!("could not persist a cache wait: {e:#}");
6067    }
6068    let remaining = budget.saturating_sub(started.elapsed());
6069    match crate::cache::wait_for(&home, cache_dir, owner, remaining, Duration::from_secs(5)).await {
6070        Ok(g) => Ok(g),
6071        Err(e) => {
6072            state.event("verify", format!("{context}: {e:#}"));
6073            if let Err(e2) = state.save() {
6074                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
6075            }
6076            Err(e)
6077        }
6078    }
6079}
6080
6081/// Run `body` — a verify command batch — while holding the shared build
6082/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
6083/// never interleave with another borrower's build against the same
6084/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
6085/// timeout, or a human's own `magi review`. See the `cache` module doc for
6086/// why this matters more than Cargo's own per-target locking covers — two
6087/// *different* worktrees building the same package name/version into one
6088/// cache directory is a staleness bug, not a lock contention one.
6089///
6090/// The wait for the lease is carved out of `budget`, never on top of it —
6091/// `body` is handed whatever is left, so a caller's own node timeout is the
6092/// only clock involved, exactly what AGENTS.md's build-cache section asks
6093/// for ("never an unbounded wait"). When `cache_dir` is `None` — no shared
6094/// cache configured at all — this is a pass-through: `body` runs with the
6095/// full budget and nothing is leased.
6096///
6097/// A lease that cannot be acquired within `budget` is reported as a single
6098/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
6099/// verification — the same shape a spawn failure already takes in
6100/// [`run_commands`], so a caller need not special-case it.
6101#[allow(clippy::too_many_arguments)]
6102async fn with_cache_lease<'s, F, Fut>(
6103    state: &'s mut RunState,
6104    cache_dir: Option<&Path>,
6105    node: &str,
6106    seat: &str,
6107    worktree: &Path,
6108    head: &str,
6109    budget: Duration,
6110    context: &str,
6111    body: F,
6112) -> (Vec<CommandOutcome>, bool)
6113where
6114    F: FnOnce(&'s mut RunState, Duration) -> Fut,
6115    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
6116{
6117    let Some(cache_dir) = cache_dir else {
6118        let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
6119        return (outcomes, retried);
6120    };
6121    let home = crate::run::home();
6122    let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
6123    let started = Instant::now();
6124    let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context).await {
6125        Ok(g) => g,
6126        Err(e) => {
6127            return (
6128                vec![CommandOutcome {
6129                    command: "(waiting for the shared build cache)".to_owned(),
6130                    code: None,
6131                    output_tail: e.to_string(),
6132                    duration_ms: started.elapsed().as_millis() as u64,
6133                    resource_blocked: true,
6134                }],
6135                false,
6136            );
6137        }
6138    };
6139    let identity = crate::cache::Identity::new(worktree, head);
6140    if let Err(e) = crate::cache::ensure_fresh(&home, cache_dir, &identity) {
6141        // A failed freshness check means this process cannot vouch for what
6142        // is sitting in the cache right now - on Windows this is exactly the
6143        // "a stale test executable is still locked, `cargo clean -p` cannot
6144        // remove it" case the evidence log records. Running verify anyway
6145        // and reporting whatever it says would let a result nobody can trust
6146        // stand for the tree it claims to have checked; fail the step
6147        // instead of the patch.
6148        state.event(
6149            "verify",
6150            format!(
6151                "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
6152                worktree.display(),
6153                short(head)
6154            ),
6155        );
6156        guard.release();
6157        return (
6158            vec![CommandOutcome {
6159                command: "(confirming the shared build cache is fresh)".to_owned(),
6160                code: None,
6161                output_tail: e.to_string(),
6162                duration_ms: started.elapsed().as_millis() as u64,
6163                resource_blocked: true,
6164            }],
6165            false,
6166        );
6167    }
6168    let remaining = budget.saturating_sub(started.elapsed());
6169    let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
6170    // A timed-out command's process was only *asked* to die (`kill_on_drop`,
6171    // `start_kill`); confirm it actually has before handing the directory to
6172    // the next acquirer. See `wait_for_timed_out_children_to_die`'s own doc
6173    // for what this can and cannot see.
6174    if !timed_out_pids.is_empty() {
6175        wait_for_timed_out_children_to_die(&timed_out_pids).await;
6176    }
6177    guard.release();
6178    (outcomes, retried)
6179}
6180
6181/// Poll `pids` — commands [`run_commands`] reports as still running when its
6182/// own timeout elapsed — until every one is confirmed gone, or
6183/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
6184///
6185/// Real confirmation where confirmation is possible, not a substitute for
6186/// full process-tree observation: a grandchild the timed-out process spawned
6187/// and that survives independently of it is invisible to a pid check the
6188/// same way it always was, and continuing to observe and collect *that*
6189/// stays a different piece of work with its own owner. This only narrows a
6190/// fixed blind wait into an actual check of the pids this process does know
6191/// about.
6192async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
6193    wait_for_pids_with(
6194        pids,
6195        crate::proc::pid_alive,
6196        LEASE_RELEASE_POLL,
6197        LEASE_RELEASE_MAX_WAIT,
6198    )
6199    .await;
6200}
6201
6202/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
6203/// interval and ceiling supplied by the caller, so the polling *logic* -
6204/// returns as soon as every pid reports dead, gives up at the ceiling
6205/// otherwise - is testable on millisecond durations without asking the real
6206/// OS about a pid at all.
6207async fn wait_for_pids_with<F: Fn(u32) -> bool>(
6208    pids: &[u32],
6209    alive: F,
6210    poll: Duration,
6211    max_wait: Duration,
6212) {
6213    let deadline = Instant::now() + max_wait;
6214    loop {
6215        if pids.iter().all(|&pid| !alive(pid)) {
6216            return;
6217        }
6218        if Instant::now() >= deadline {
6219            return;
6220        }
6221        tokio::time::sleep(poll).await;
6222    }
6223}
6224
6225/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
6226/// admission that it could not even get a verify command to run, as opposed
6227/// to evidence the command actually produced? A caller that would otherwise
6228/// read a resource-blocked outcome as a red command must check this first:
6229/// see [`Runner::gate`], which retries rather than records `Blocked` when
6230/// this is true.
6231fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
6232    outcomes.iter().any(|o| o.resource_blocked)
6233}
6234
6235/// What [`Runner::gate_fix_round`] decided.
6236enum GateFix {
6237    /// The tree changed and `verify.e2e` is still green: run the gate again.
6238    Retry,
6239    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
6240    /// last failure stands and the run ends blocked.
6241    Stop,
6242    /// `verify.e2e` could not run after the fix (magi's own contention):
6243    /// decide nothing now, a later reentry retries.
6244    Defer,
6245}
6246
6247/// Is every red command in `outcomes` an ordinary failure the code could
6248/// explain: it ran, exited non-zero, and said something?
6249///
6250/// A timeout, a spawn failure and a killed process all leave `code` `None`;
6251/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
6252/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
6253/// what the command is stays the gate's business.
6254fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
6255    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
6256    red.peek().is_some()
6257        && red.all(|o| {
6258            !o.resource_blocked
6259                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
6260                && !o.output_tail.trim().is_empty()
6261        })
6262}
6263
6264/// Describe one verify command's outcome for the event log, distinguishing a
6265/// build/link failure — the toolchain never produced a binary to run — from
6266/// an actual test failure, since only the latter is a verdict on the patch.
6267fn e2e_outcome_label(o: &CommandOutcome) -> String {
6268    if o.ok() {
6269        return "pass".to_owned();
6270    }
6271    let reason = if o.build_failed() {
6272        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
6273    } else {
6274        format!("FAIL ({:?})", o.code)
6275    };
6276    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
6277}
6278
6279/// Run `verify.e2e`, retrying once if the first attempt could not build or
6280/// link — a build/link failure is frequently a race against a shared
6281/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
6282/// `verify` event per command, tagged with `context` (normally `"round N"`)
6283/// so the two call sites that need this — the ordinary per-round leg in
6284/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
6285/// it will ever call a round green — read identically in the event log.
6286async fn run_e2e_with_retry(
6287    state: &mut RunState,
6288    shell: &[String],
6289    commands: &[String],
6290    worktree: &Path,
6291    timeout: Duration,
6292    context: &str,
6293) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
6294    let (mut e2e, mut timed_out_pids) = run_commands(
6295        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
6296    )
6297    .await;
6298    for o in &e2e {
6299        state.event(
6300            "verify",
6301            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
6302        );
6303    }
6304    // A build/link failure is not a verdict on the patch — it is frequently a
6305    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
6306    // one retry before letting a red like that decide the round.
6307    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
6308    if verify_retried {
6309        state.event(
6310            "verify",
6311            format!(
6312                "{context}: verify could not build/link, not a test result — retrying once \
6313                 before concluding"
6314            ),
6315        );
6316        let retried = run_commands(
6317            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
6318        )
6319        .await;
6320        e2e = retried.0;
6321        // Both attempts' timeouts matter, not just the last one: the first
6322        // attempt's descendants may still be alive alongside the retry's.
6323        timed_out_pids.extend(retried.1);
6324        for o in &e2e {
6325            state.event(
6326                "verify",
6327                format!(
6328                    "{context}: retry `{}` -> {}",
6329                    o.command,
6330                    e2e_outcome_label(o)
6331                ),
6332            );
6333        }
6334    }
6335    (e2e, verify_retried, timed_out_pids)
6336}
6337
6338/// Run configured shell commands in `cwd`, in order. The second element is
6339/// the pid of every command that hit `timeout` and was still running when
6340/// this stopped waiting on it (best-effort: `None` when the platform did not
6341/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
6342/// that releases a shared resource afterward needs to know.
6343///
6344/// Records `task` into [`RunState::active`] at every command boundary
6345/// (`RunState::task_command`) and clears it once the whole list has run
6346/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
6347/// for minutes with no seat and no output of its own to show for it (see
6348/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
6349/// apart from "not yet run" without this), and this is the only place that
6350/// knows which command is running right now and how many are left. Three
6351/// saves per command — start, not per second — matching the same "only at a
6352/// boundary" rule [`wave`] already follows for seats.
6353#[allow(clippy::too_many_arguments)]
6354async fn run_commands(
6355    state: &mut RunState,
6356    node: &str,
6357    task: &str,
6358    attempt: usize,
6359    shell: &[String],
6360    commands: &[String],
6361    cwd: &Path,
6362    timeout: Duration,
6363) -> (Vec<CommandOutcome>, Vec<u32>) {
6364    if commands.is_empty() {
6365        // Nothing to mark as running and nothing to clear — an empty list
6366        // means "not configured", and touching `active` (or the disk) over
6367        // that would be a write for every round of a repo with no
6368        // `verify.e2e` / `verify.gate` commands at all.
6369        return (Vec::new(), Vec::new());
6370    }
6371    let mut out = Vec::new();
6372    let mut timed_out_pids = Vec::new();
6373    let total = commands.len();
6374    for (idx, command) in commands.iter().enumerate() {
6375        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
6376        if let Err(e) = state.save() {
6377            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
6378        }
6379        let started = Instant::now();
6380        let mut cmd = tokio::process::Command::new(&shell[0]);
6381        cmd.quiet();
6382        cmd.args(&shell[1..])
6383            .arg(command)
6384            .current_dir(cwd)
6385            .stdin(std::process::Stdio::null())
6386            .stdout(std::process::Stdio::piped())
6387            .stderr(std::process::Stdio::piped())
6388            .kill_on_drop(true);
6389        let spawned = cmd.spawn();
6390        let (code, body) = match spawned {
6391            Ok(child) => {
6392                // Captured before the child is consumed below: `kill_on_drop`
6393                // only *asks* the process to die when the timeout branch
6394                // drops it, and the pid is the only way anyone downstream can
6395                // later check whether that request actually took.
6396                let pid = child.id();
6397                match tokio::time::timeout(timeout, child.wait_with_output()).await {
6398                    Ok(Ok(o)) => {
6399                        let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
6400                        body.push_str(&String::from_utf8_lossy(&o.stderr));
6401                        (o.status.code(), body)
6402                    }
6403                    Ok(Err(e)) => (None, format!("failed to run: {e}")),
6404                    Err(_) => {
6405                        if let Some(pid) = pid {
6406                            timed_out_pids.push(pid);
6407                        }
6408                        (None, format!("timed out after {}s", timeout.as_secs()))
6409                    }
6410                }
6411            }
6412            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
6413        };
6414        out.push(CommandOutcome {
6415            command: command.clone(),
6416            code,
6417            output_tail: tail(&body, OUTPUT_TAIL),
6418            duration_ms: started.elapsed().as_millis() as u64,
6419            resource_blocked: false,
6420        });
6421    }
6422    state.task_finished(task);
6423    if let Err(e) = state.save() {
6424        tracing::warn!("could not persist the end of {task}: {e:#}");
6425    }
6426    (out, timed_out_pids)
6427}
6428
6429/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
6430/// section (`report::run`) and in the `merge` event this node records — for
6431/// the operator to run by hand.
6432///
6433/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
6434/// branch whose ruleset forbids merge commits (GitHub's "must not contain
6435/// merge commits", or "require linear history") rejects the push a `--no-ff`
6436/// merge would produce, which is exactly the guidance this function replaces.
6437/// `message`'s first line becomes the squash commit's subject, matching the
6438/// note `report::run` prints alongside this command — see that function for
6439/// why an explicit subject is not optional there.
6440fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
6441    let repo = repo.display();
6442    match style {
6443        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
6444        MergeStyle::Squash => {
6445            // The subject sits inside double quotes, and a title an agent
6446            // wrote may carry the characters that break out of them.
6447            let subject = message
6448                .lines()
6449                .next()
6450                .unwrap_or(branch)
6451                .replace(['\\', '"', '$', '`'], "");
6452            format!(
6453                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
6454            )
6455        }
6456        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
6457    }
6458}
6459
6460/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
6461/// under the hood, rejects a `title` over 256 characters and the whole
6462/// command fails — no PR at all, for a run whose body was otherwise fine
6463/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
6464/// that limit: `title_from` counts `chars()` (Unicode scalars), which is not
6465/// always how GitHub counts, plus one character for the trailing ellipsis
6466/// `title_from` may add. It is a margin, not a guarantee — a title packed
6467/// with multi-unit characters could still in principle land close to the
6468/// edge, but a real task title's occasional emoji or accented letter fits
6469/// comfortably inside it.
6470const PR_TITLE_MAX: usize = 240;
6471
6472/// What `merge = "pr"` (and the merge commit of the other modes) says about a
6473/// change: a title and a body describing what was *implemented*, not the task
6474/// that asked for it. A task reads as a request; a reader of the merged
6475/// history wants the change.
6476struct PrMessage {
6477    title: String,
6478    body: String,
6479}
6480
6481impl PrMessage {
6482    /// Title, blank line, body. The first line is the squash/merge commit
6483    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
6484    /// has to stay one sensible line.
6485    fn commit_message(&self) -> String {
6486        format!("{}\n\n{}", self.title, self.body)
6487    }
6488}
6489
6490/// The text after a leading `TITLE:` (any case) on `line`.
6491fn title_marker(line: &str) -> Option<&str> {
6492    let line = line.trim();
6493    let head = line.get(..6)?;
6494    head.eq_ignore_ascii_case("title:")
6495        .then(|| line[6..].trim())
6496}
6497
6498/// The implementer's own one-line title: the `TITLE:` line the implement
6499/// prompt asks for at the top of its SUMMARY. Candidate commits are all
6500/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
6501/// source, and a title that says as much is refused here too.
6502fn summary_title(summary: &str) -> Option<String> {
6503    let first = summary.lines().find(|l| !l.trim().is_empty())?;
6504    let raw = title_marker(first)?;
6505    if raw.is_empty() {
6506        return None;
6507    }
6508    let title = queue::title_from(raw, PR_TITLE_MAX);
6509    let lower = title.to_ascii_lowercase();
6510    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
6511        return None;
6512    }
6513    Some(title)
6514}
6515
6516/// `summary` without its `TITLE:` line, which the pull request title already
6517/// carries.
6518fn summary_without_title(summary: &str) -> String {
6519    let mut lines = summary.trim().lines().peekable();
6520    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
6521        lines.next();
6522    }
6523    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
6524}
6525
6526/// The pull request title and body for the winning candidate.
6527///
6528/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
6529/// the task's own opening line via [`queue::title_from`] when there is none.
6530/// `state.instruction` can open with blank lines (`task_text` only rejects a
6531/// body that is blank *entirely*), which `title_from` skips.
6532///
6533/// Body: the implementer's summary and the fixer's notes, then — when the
6534/// winning review round was not clean — the findings still open and whatever
6535/// the fixer declined, so `merge = "pr"` hands the reader the same material
6536/// `magi show` does. The task follows inside a collapsed block, and the
6537/// footer repeats the run and candidate as plain tags for a reader holding
6538/// only the merged commit or the PR body.
6539fn pr_message(state: &RunState, winner: char) -> PrMessage {
6540    let summary = state
6541        .candidates
6542        .iter()
6543        .find(|c| c.label == winner)
6544        .map(|c| c.summary.as_str())
6545        .unwrap_or_default();
6546    // The fallback is the operator's own words and may not be English; GitHub
6547    // text always is, so a non-English task gets a neutral title instead.
6548    let title = summary_title(summary).unwrap_or_else(|| {
6549        let t = queue::title_from(&state.instruction, PR_TITLE_MAX);
6550        if t.is_ascii() && t.chars().any(|c| c.is_ascii_alphabetic()) {
6551            t
6552        } else {
6553            format!(
6554                "chore: land candidate {} of run {}",
6555                winner.to_ascii_uppercase(),
6556                state.id
6557            )
6558        }
6559    });
6560
6561    let mut body = String::new();
6562    let what = summary_without_title(summary);
6563    if !what.is_empty() {
6564        body.push_str("## Summary\n\n");
6565        body.push_str(&what);
6566        body.push_str("\n\n");
6567    }
6568
6569    let fix = state.reviews.last().and_then(|r| r.fix.as_ref());
6570    if let Some(fix) = fix
6571        && !fix.notes.trim().is_empty()
6572    {
6573        body.push_str("## Review fixes\n\n");
6574        body.push_str(fix.notes.trim());
6575        body.push_str("\n\n");
6576    }
6577
6578    let open = state.open_findings();
6579    if !open.is_empty() {
6580        body.push_str("## Open review findings\n\n");
6581        for f in &open {
6582            body.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
6583        }
6584        body.push('\n');
6585    }
6586
6587    if let Some(fix) = fix
6588        && !fix.rejected.is_empty()
6589    {
6590        body.push_str("## Declined by the fixer\n\n");
6591        for r in &fix.rejected {
6592            body.push_str(&format!("- `{}`: {}\n", r.id, r.why));
6593        }
6594        body.push('\n');
6595    }
6596
6597    let task = state.instruction.trim();
6598    let task = if task.is_empty() {
6599        "(empty task)"
6600    } else {
6601        task
6602    };
6603    body.push_str(&format!(
6604        "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
6605        task.replace("</details>", "&lt;/details&gt;")
6606    ));
6607
6608    body.push_str(&format!(
6609        "\n---\nmagi:run/{} magi:candidate-{}\n",
6610        state.id,
6611        winner.to_ascii_lowercase()
6612    ));
6613
6614    // Prompts are advisory; this is the enforced half of the confidentiality
6615    // rule, and it covers the verbatim task in <details> too.
6616    let id = crate::scrub::Identity::current();
6617    PrMessage {
6618        title: crate::scrub::scrub(&title, &id),
6619        body: crate::scrub::scrub(&body, &id),
6620    }
6621}
6622
6623/// `gh pr create`, returning the PR url.
6624async fn gh_pr_create(
6625    cwd: &Path,
6626    base: &str,
6627    head: &str,
6628    title: &str,
6629    body: &str,
6630) -> Result<String> {
6631    let out = tokio::process::Command::new("gh")
6632        .args([
6633            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
6634        ])
6635        .current_dir(cwd)
6636        .quiet()
6637        .stdin(std::process::Stdio::null())
6638        .output()
6639        .await
6640        .context("spawn gh")?;
6641    if out.status.success() {
6642        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
6643    } else {
6644        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
6645    }
6646}
6647
6648/// Tear a run's worktrees and branches down.
6649///
6650/// `home` is where the updated `run.json` is saved (via
6651/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
6652/// a housekeeping pass already has its own honest `home` handed to it, and
6653/// falling through to the global here would write back through whichever
6654/// directory some other process or test pinned into that `OnceLock` first,
6655/// not the one the caller actually resolved its `runs` and `state` from.
6656pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
6657    let repo = state.repo.clone();
6658    let root = state.worktree_root();
6659    let winner = state.tally.as_ref().map(|t| t.winner);
6660    let mut removed = Vec::new();
6661
6662    for i in 0..state.candidates.len() {
6663        let c = state.candidates[i].clone();
6664        let is_winner = Some(c.label) == winner;
6665        if is_winner && !drop_winner {
6666            continue;
6667        }
6668        if c.worktree.exists() {
6669            git::worktree_remove(&repo, &c.worktree).await.ok();
6670            removed.push(c.worktree.to_string_lossy().into_owned());
6671        }
6672        if git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
6673            git::branch_delete(&repo, &c.branch).await.ok();
6674            removed.push(c.branch.clone());
6675        }
6676        state.candidates[i].folded = true;
6677    }
6678
6679    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
6680        let path = name.path();
6681        let keep = !drop_winner
6682            && winner.is_some_and(|w| {
6683                path.file_name()
6684                    .is_some_and(|n| n == format!("cand-{w}").as_str())
6685            });
6686        if keep {
6687            continue;
6688        }
6689        git::worktree_remove(&repo, &path).await.ok();
6690        removed.push(path.to_string_lossy().into_owned());
6691    }
6692
6693    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
6694    // judge worktrees, so once the loop above has cleared all of them out,
6695    // the parent is a bare directory nobody else was ever going to remove -
6696    // git only ever managed what was inside it. Left alone, one of these
6697    // accumulates per fully-folded run; the operator's own machine had 74.
6698    // `remove_if_empty` re-checks rather than assuming: a run whose winner
6699    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
6700    // so does anything a run never claimed that happens to share the bay.
6701    remove_if_empty(&root);
6702
6703    if state.enabled_worktree_config && drop_winner {
6704        // A release, not a raw disable: some sibling run in this repository
6705        // may still hold its own reference (see `git::acquire_worktree_config`),
6706        // and only the last release actually turns the setting back off.
6707        git::release_worktree_config(&repo).await.ok();
6708        state.enabled_worktree_config = false;
6709    }
6710    state.save_under(home)?;
6711    Ok(removed)
6712}
6713
6714/// Remove `dir` if it exists and has nothing in it.
6715///
6716/// Best-effort and silent by design: a directory that is not empty (a run
6717/// whose winner is still parked there, a stray file some other process left)
6718/// is exactly the case this must refuse, and a directory that is already gone
6719/// is not a failure worth reporting either. `std::fs::remove_dir` itself
6720/// already refuses a non-empty directory, so the emptiness check below is
6721/// belt, not suspenders - it is what keeps this from ever attempting the
6722/// removal in the case that matters, rather than trusting `remove_dir`'s
6723/// error path to have no side effects if it ever changed.
6724fn remove_if_empty(dir: &Path) {
6725    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
6726        std::fs::remove_dir(dir).ok();
6727    }
6728}
6729
6730/// Severity of the worst open finding in the last review round, for reporting.
6731pub fn worst_open(state: &RunState) -> Option<Severity> {
6732    state
6733        .reviews
6734        .last()?
6735        .reviews
6736        .iter()
6737        .flat_map(|r| r.findings.iter())
6738        .map(|f| f.severity)
6739        .max()
6740}
6741
6742#[cfg(test)]
6743mod tests {
6744    use super::*;
6745    use crate::run::GateStatus;
6746    use std::collections::BTreeMap;
6747    use std::time::Duration;
6748
6749    fn conductor() -> AgentSpec {
6750        AgentSpec {
6751            id: "conductor".to_owned(),
6752            kind: crate::config::AgentKind::Command,
6753            model: None,
6754            command: vec!["true".to_owned()],
6755            extra_args: Vec::new(),
6756            env: BTreeMap::new(),
6757            prompt_delivery: None,
6758        }
6759    }
6760
6761    fn spec(id: &str) -> AgentSpec {
6762        AgentSpec {
6763            id: id.to_owned(),
6764            kind: crate::config::AgentKind::Command,
6765            model: None,
6766            command: vec!["true".to_owned()],
6767            extra_args: Vec::new(),
6768            env: BTreeMap::new(),
6769            prompt_delivery: None,
6770        }
6771    }
6772
6773    // `next_untried_implementer` is the property `resume_quota_losses`'s own
6774    // fallback loop depends on to terminate: it must walk forward from the
6775    // seat's own position, never restart at the front of the roster, and it
6776    // must never hand back an id already tried, however many times that id
6777    // happens to appear.
6778
6779    #[test]
6780    fn next_untried_implementer_walks_forward_from_the_seats_own_position() {
6781        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
6782        let tried = BTreeSet::from(["beta".to_owned()]);
6783        // beta sits at index 1; the next candidate is gamma, never alpha —
6784        // which is very likely a different candidate slot's own agent.
6785        let next = next_untried_implementer(&roster, 1, &tried);
6786        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
6787    }
6788
6789    #[test]
6790    fn next_untried_implementer_does_not_wrap_back_past_its_own_start() {
6791        let roster = vec![spec("alpha"), spec("beta")];
6792        let tried = BTreeSet::from(["beta".to_owned()]);
6793        // beta is the roster's last entry: nothing follows it, and alpha —
6794        // earlier in the roster, almost certainly a different candidate
6795        // slot's own agent — must not be reached by wrapping back to it.
6796        assert!(next_untried_implementer(&roster, 1, &tried).is_none());
6797    }
6798
6799    #[test]
6800    fn next_untried_implementer_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
6801        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
6802        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
6803        // beta (index 1) and gamma (index 2, the only entry after it) have
6804        // both been tried; alpha (index 0) never has, but it comes before
6805        // beta's own position, so there is nothing further for this seat.
6806        assert!(next_untried_implementer(&roster, 1, &tried).is_none());
6807    }
6808
6809    #[test]
6810    fn next_untried_implementer_skips_ids_already_tried_even_when_duplicated() {
6811        let roster = vec![spec("a"), spec("a"), spec("b")];
6812        let tried = BTreeSet::from(["a".to_owned()]);
6813        let next = next_untried_implementer(&roster, 0, &tried);
6814        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
6815    }
6816
6817    #[test]
6818    fn next_untried_implementer_returns_none_once_every_id_is_tried() {
6819        let roster = vec![spec("a"), spec("b")];
6820        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
6821        assert!(next_untried_implementer(&roster, 0, &tried).is_none());
6822    }
6823
6824    #[test]
6825    fn remove_if_empty_only_ever_takes_a_bare_directory() {
6826        let dir = tempfile::tempdir().unwrap();
6827        let bay = dir.path().join("ffff");
6828
6829        // Not there yet: nothing to do, nothing to panic on.
6830        remove_if_empty(&bay);
6831        assert!(!bay.exists());
6832
6833        // Something still inside - the winner's worktree, or a stray file -
6834        // keeps the directory standing.
6835        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
6836        remove_if_empty(&bay);
6837        assert!(bay.exists(), "non-empty directory must survive");
6838
6839        // Once the last entry is gone, so is the directory itself.
6840        std::fs::remove_dir(bay.join("cand-A")).unwrap();
6841        remove_if_empty(&bay);
6842        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
6843    }
6844
6845    // `round_is_clean` is the exact decision this task fixed: a round with a
6846    // seat that never answered must not read the same as a round every seat
6847    // actually reviewed. These are deterministic and process-free by design —
6848    // the equivalent end-to-end check (a real reviewer timing out under a
6849    // live graph run) is a genuine race against wall-clock contention, and a
6850    // spawn slow enough to blow even a generous budget under a loaded test
6851    // run must not turn this specific regression check flaky.
6852
6853    #[test]
6854    fn a_full_panel_that_found_nothing_is_clean() {
6855        assert!(round_is_clean(
6856            0,
6857            true,
6858            2,
6859            2,
6860            0,
6861            IncompleteReviewPolicy::Block
6862        ));
6863    }
6864
6865    #[test]
6866    fn a_missing_seat_is_never_clean_under_the_default_policy() {
6867        assert!(!round_is_clean(
6868            0,
6869            true,
6870            1,
6871            2,
6872            0,
6873            IncompleteReviewPolicy::Block
6874        ));
6875    }
6876
6877    #[test]
6878    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
6879        assert!(!round_is_clean(
6880            1,
6881            true,
6882            1,
6883            2,
6884            0,
6885            IncompleteReviewPolicy::Warn
6886        ));
6887    }
6888
6889    #[test]
6890    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
6891        assert!(round_is_clean(
6892            0,
6893            true,
6894            1,
6895            2,
6896            0,
6897            IncompleteReviewPolicy::Warn
6898        ));
6899    }
6900
6901    #[test]
6902    fn a_full_panel_with_an_open_finding_is_not_clean() {
6903        assert!(!round_is_clean(
6904            1,
6905            true,
6906            2,
6907            2,
6908            0,
6909            IncompleteReviewPolicy::Block
6910        ));
6911    }
6912
6913    #[test]
6914    fn a_full_panel_with_a_red_e2e_is_not_clean() {
6915        assert!(!round_is_clean(
6916            0,
6917            false,
6918            2,
6919            2,
6920            0,
6921            IncompleteReviewPolicy::Block
6922        ));
6923    }
6924
6925    // The stall this task closes: under the default `block` policy, a seat
6926    // missing only because it was rate limited must not force a wait for a
6927    // session limit that will not lift by the next round. `round_is_clean`
6928    // is where that quorum carve-out lives; the review loop around it never
6929    // changes what a reviewer's vote or a finding's severity means.
6930
6931    #[test]
6932    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
6933        // 1 of 2 answered, and the one missing was quota'd — the exact
6934        // "review-2 rate limited (quota)" shape from the field report.
6935        assert!(round_is_clean(
6936            0,
6937            true,
6938            1,
6939            2,
6940            1,
6941            IncompleteReviewPolicy::Block
6942        ));
6943    }
6944
6945    #[test]
6946    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
6947        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
6948        // not a quota loss (`quota_missing` stays 0) — worth another try.
6949        assert!(!round_is_clean(
6950            0,
6951            true,
6952            1,
6953            2,
6954            0,
6955            IncompleteReviewPolicy::Block
6956        ));
6957    }
6958
6959    #[test]
6960    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
6961        assert!(!round_is_clean(
6962            1,
6963            true,
6964            1,
6965            2,
6966            1,
6967            IncompleteReviewPolicy::Block
6968        ));
6969        assert!(!round_is_clean(
6970            0,
6971            false,
6972            1,
6973            2,
6974            1,
6975            IncompleteReviewPolicy::Block
6976        ));
6977    }
6978
6979    #[test]
6980    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
6981        // Every seat quota'd, nobody answered: there is no panel to decide
6982        // on, so this must fall through to the existing block-and-retry
6983        // fallback rather than call an unreviewed patch clean.
6984        assert!(!round_is_clean(
6985            0,
6986            true,
6987            0,
6988            2,
6989            2,
6990            IncompleteReviewPolicy::Block
6991        ));
6992    }
6993
6994    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
6995        CommandOutcome {
6996            command: "test".to_owned(),
6997            code,
6998            output_tail: String::new(),
6999            duration_ms: 0,
7000            resource_blocked,
7001        }
7002    }
7003
7004    #[test]
7005    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
7006        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
7007        assert!(
7008            !verify_inconclusive(&[outcome(Some(1), false)]),
7009            "an ordinary failure is still evidence about the patch"
7010        );
7011        assert!(verify_inconclusive(&[outcome(None, true)]));
7012        assert!(
7013            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
7014            "one inconclusive outcome taints the whole batch"
7015        );
7016        assert!(!verify_inconclusive(&[]));
7017    }
7018
7019    #[tokio::test]
7020    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
7021        // Alive for the first two checks, then dead - confirms the loop
7022        // actually re-polls rather than deciding once and sleeping out the
7023        // ceiling regardless.
7024        let calls = std::sync::atomic::AtomicUsize::new(0);
7025        let started = Instant::now();
7026        wait_for_pids_with(
7027            &[123],
7028            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
7029            Duration::from_millis(5),
7030            Duration::from_secs(5),
7031        )
7032        .await;
7033        assert!(
7034            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
7035            "must keep checking rather than deciding on the first answer"
7036        );
7037        assert!(
7038            started.elapsed() < Duration::from_secs(1),
7039            "must return the moment it is confirmed dead, not wait out the ceiling"
7040        );
7041    }
7042
7043    #[tokio::test]
7044    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
7045        let started = Instant::now();
7046        wait_for_pids_with(
7047            &[123],
7048            |_| true, // never reports dead
7049            Duration::from_millis(5),
7050            Duration::from_millis(30),
7051        )
7052        .await;
7053        let elapsed = started.elapsed();
7054        assert!(
7055            elapsed >= Duration::from_millis(30),
7056            "must not give up before its own ceiling: {elapsed:?}"
7057        );
7058        assert!(
7059            elapsed < Duration::from_secs(1),
7060            "must not wait past its own ceiling either: {elapsed:?}"
7061        );
7062    }
7063
7064    #[tokio::test]
7065    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
7066        let started = Instant::now();
7067        wait_for_pids_with(
7068            &[],
7069            |_| true,
7070            Duration::from_secs(5),
7071            Duration::from_secs(5),
7072        )
7073        .await;
7074        assert!(
7075            started.elapsed() < Duration::from_millis(200),
7076            "an empty pid list has nothing to confirm"
7077        );
7078    }
7079
7080    // `review_conclusion` is the exact decision the review hand-off task
7081    // fixed: a round budget spent (or a tree that stopped moving) must not
7082    // collapse into `Blocked` regardless of what verification actually
7083    // said. Deterministic and process-free for the same reason the
7084    // `round_is_clean` family above is.
7085    fn review_round(
7086        clean: bool,
7087        blocking: usize,
7088        answered: usize,
7089        expected: usize,
7090        progressed: bool,
7091        e2e_ok: bool,
7092    ) -> ReviewRound {
7093        ReviewRound {
7094            round: 1,
7095            head: "h".to_owned(),
7096            verified_head: None,
7097            verified_at: None,
7098            reviews: Vec::new(),
7099            e2e: vec![CommandOutcome {
7100                command: "test".to_owned(),
7101                code: Some(if e2e_ok { 0 } else { 1 }),
7102                output_tail: String::new(),
7103                duration_ms: 0,
7104                resource_blocked: false,
7105            }],
7106            verify_retried: false,
7107            e2e_deferred: false,
7108            e2e_defer_reason: None,
7109            fix: None,
7110            blocking,
7111            answered,
7112            expected,
7113            clean,
7114            progressed,
7115            vote_split: false,
7116            reconsideration: Vec::new(),
7117            verdict: None,
7118        }
7119    }
7120
7121    #[test]
7122    fn review_conclusion_is_none_when_nothing_has_run() {
7123        assert_eq!(review_conclusion(&[], 3), None);
7124    }
7125
7126    #[test]
7127    fn review_conclusion_is_none_while_rounds_remain() {
7128        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
7129        assert_eq!(review_conclusion(&rounds, 3), None);
7130    }
7131
7132    #[test]
7133    fn review_conclusion_is_gating_once_a_round_is_clean() {
7134        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
7135        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
7136    }
7137
7138    #[test]
7139    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
7140        let rounds = vec![
7141            review_round(false, 1, 2, 2, true, true),
7142            review_round(false, 1, 2, 2, true, true),
7143        ];
7144        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
7145    }
7146
7147    #[test]
7148    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
7149        let rounds = vec![
7150            review_round(false, 1, 2, 2, true, true),
7151            review_round(false, 1, 2, 2, true, false),
7152        ];
7153        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
7154    }
7155
7156    #[test]
7157    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
7158        // Magi never got a command to run against this round's own head — a
7159        // resource-blocked attempt, not a red one — so this must never
7160        // settle on `Blocked` the way a genuine e2e failure would. `None`
7161        // here is what tells `Runner::review_loop` to retry the check
7162        // itself rather than trust this cheap recomputation with a verdict
7163        // it cannot actually produce.
7164        let mut blocked = review_round(false, 1, 2, 2, true, false);
7165        blocked.e2e[0].resource_blocked = true;
7166        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
7167        assert_eq!(review_conclusion(&rounds, 2), None);
7168    }
7169
7170    #[test]
7171    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
7172        // Missing input, not a verified tree — never a hand-off candidate.
7173        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
7174        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
7175    }
7176
7177    #[test]
7178    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
7179        let rounds = vec![
7180            review_round(false, 1, 2, 2, false, true),
7181            review_round(false, 1, 2, 2, false, true),
7182        ];
7183        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
7184    }
7185
7186    fn secs(n: u64) -> Duration {
7187        Duration::from_secs(n)
7188    }
7189
7190    /// A throwaway repo with one commit on `main`, for tests that need `merge`
7191    /// to make real (and, if it runs at all, real*ly fail*) git calls.
7192    fn init_repo(dir: &Path) {
7193        let run = |args: &[&str]| {
7194            let out = std::process::Command::new("git")
7195                .args(args)
7196                .current_dir(dir)
7197                .quiet()
7198                .output()
7199                .expect("spawn git");
7200            assert!(
7201                out.status.success(),
7202                "git {args:?} failed: {}",
7203                String::from_utf8_lossy(&out.stderr)
7204            );
7205        };
7206        run(&["init", "-b", "main"]);
7207        run(&["config", "user.name", "magi test"]);
7208        run(&["config", "user.email", "magi@example.com"]);
7209        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
7210        run(&["add", "-A"]);
7211        run(&["commit", "-m", "init"]);
7212    }
7213
7214    // `settle_questions` is what closes the ghost the phone showed: a run's
7215    // seat asked something, the run then ended, and nothing was left to
7216    // abandon the question it left `open`. `HOME` is a process-wide
7217    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
7218    // first time it runs in the binary — every test below still reaches the
7219    // same directory whichever call won, and each gets its own run id from
7220    // `RunState::new`, so they never collide there.
7221    fn ask_test_home() {
7222        crate::run::set_home(std::env::temp_dir().join("magi-graph-ask-tests-home"));
7223    }
7224
7225    /// A minimal, git-free `Runner` at a given status — `settle_questions`
7226    /// reads nothing else off it.
7227    fn runner_at(status: RunStatus) -> Runner {
7228        let mut state = RunState::new(
7229            PathBuf::from("/nonexistent/repo"),
7230            "main".to_owned(),
7231            "deadbeef".to_owned(),
7232            "task".to_owned(),
7233            Config::default(),
7234        );
7235        state.status = status;
7236        Runner {
7237            state,
7238            roles: ResolvedRoles {
7239                implementers: Vec::new(),
7240                judges: Vec::new(),
7241                reviewers: Vec::new(),
7242                fixer: None,
7243                conductor: conductor(),
7244                implementer_roster: Vec::new(),
7245            },
7246            sem: Arc::new(Semaphore::new(1)),
7247            pause: Pause::new(),
7248            interrupt: Pause::new(),
7249        }
7250    }
7251
7252    /// `park_here` folding in the reason `Pause::park_because` recorded -
7253    /// this is what lets an operator reading a run's events tell an
7254    /// interrupt-driven park from an ordinary shutdown park.
7255    #[test]
7256    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
7257        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
7258        let mut runner = runner_at(RunStatus::Implementing);
7259        let interrupt = Pause::new();
7260        runner.watch_interrupt(interrupt.clone());
7261
7262        interrupt.park_because("task a1b2 asked to run first");
7263
7264        assert!(runner.park_here().expect("park_here"));
7265        assert!(runner.state.parked);
7266        let last = runner.state.events.last().expect("a park event");
7267        assert_eq!(last.node, "park");
7268        assert!(
7269            last.message.contains("task a1b2 asked to run first"),
7270            "expected the interrupt reason in {:?}",
7271            last.message
7272        );
7273    }
7274
7275    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
7276    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
7277    /// cleared) must not make a *different* run - one only watching its own,
7278    /// unshared interrupt `Pause` - see itself as parked. If a future change
7279    /// ever collapsed these back into one handle, the interrupt scheduler
7280    /// would park every run for the rest of the daemon's life, not just the
7281    /// one it meant to interrupt.
7282    #[test]
7283    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
7284        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
7285        let mut runner = runner_at(RunStatus::Implementing);
7286        let shutdown = Pause::new();
7287        runner.on_pause(shutdown.clone());
7288        let interrupt = Pause::new();
7289        runner.watch_interrupt(interrupt.clone());
7290
7291        // Nobody has asked for anything yet.
7292        assert!(!runner.park_here().expect("park_here"));
7293        assert!(!runner.state.parked);
7294
7295        // Only the interrupt handle fires; the shutdown handle stays clear.
7296        interrupt.park_because("test");
7297        assert!(!shutdown.parked());
7298        assert!(runner.park_here().expect("park_here"));
7299    }
7300
7301    /// The property every prior attempt at this feature failed to pin down:
7302    /// asking a run to park while one of its nodes has a real, in-flight
7303    /// async operation running (an agent call, in production) must not cut
7304    /// that operation short. `park_here` is only ever consulted *between*
7305    /// `execute`'s node calls - see its own doc - so nothing inside a node
7306    /// can observe a park request until the node itself returns. This proves
7307    /// that structurally, with real `tokio` concurrency and a channel
7308    /// handshake (never a sleep, which would only prove "usually", not
7309    /// "cannot"): the "node" below reports that it has genuinely started,
7310    /// and only then is the park requested; the node still has to be told to
7311    /// finish before `park_here` is ever called, exactly mirroring every
7312    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
7313    /// in `execute`.
7314    #[tokio::test]
7315    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
7316        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
7317        let mut runner = runner_at(RunStatus::Implementing);
7318        let interrupt = Pause::new();
7319        runner.watch_interrupt(interrupt.clone());
7320
7321        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
7322        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
7323
7324        // Stands in for one node's in-flight agent call: it proves it has
7325        // genuinely started, then blocks - exactly as a spawned CLI process
7326        // does - until told to finish.
7327        let node = async move {
7328            started_tx.send(()).expect("send started");
7329            finish_rx.await.expect("recv finish");
7330            "node finished"
7331        };
7332
7333        let interrupter = async move {
7334            started_rx.await.expect("recv started");
7335            // The call is now genuinely in flight. Ask it to park.
7336            interrupt.park_because("higher-priority task waiting");
7337            // Nothing the node does can observe this yet - there is no
7338            // check inside it, by construction - so let the executor run
7339            // anything pending and then let the node finish on its own.
7340            tokio::task::yield_now().await;
7341            finish_tx.send(()).expect("send finish");
7342        };
7343
7344        let (node_result, ()) = tokio::join!(node, interrupter);
7345        assert_eq!(
7346            node_result, "node finished",
7347            "the in-flight call ran to completion"
7348        );
7349
7350        // Only now, at the boundary the real `execute` would check right
7351        // after this node, does the park take effect.
7352        assert!(runner.park_here().expect("park_here"));
7353        assert!(runner.state.parked);
7354    }
7355
7356    /// A run parked mid-competition carries every field it had accumulated
7357    /// through the exact same disk round-trip an ordinary resume uses -
7358    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
7359    /// Nothing about parking for an interrupt is a special case of that path;
7360    /// this is what proves it rather than assuming it.
7361    #[test]
7362    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
7363        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
7364        let mut runner = runner_at(RunStatus::Judging);
7365        // `Runner::resume` re-resolves roles from the saved config, which
7366        // refuses an empty roster - give it the same minimal one `conductor`
7367        // itself uses.
7368        runner.state.config.agents = vec![conductor()];
7369        runner.state.candidates = vec![Candidate {
7370            index: 0,
7371            label: 'A',
7372            agent: "alpha".to_owned(),
7373            branch: "magi/x/A".to_owned(),
7374            worktree: PathBuf::from("/nonexistent/worktree"),
7375            summary: "did the thing".to_owned(),
7376            stat: "1 file changed".to_owned(),
7377            files: 1,
7378            commits: 1,
7379            empty: false,
7380            failed: None,
7381            verified_noop: None,
7382            duration_ms: 1234,
7383            folded: false,
7384        }];
7385        let run_id = runner.state.id.clone();
7386
7387        let interrupt = Pause::new();
7388        runner.watch_interrupt(interrupt.clone());
7389        interrupt.park_because("task c3d4 asked to run first");
7390        assert!(runner.park_here().expect("park_here"));
7391
7392        let resumed = Runner::resume(&run_id).expect("resume");
7393        assert_eq!(resumed.state.candidates.len(), 1);
7394        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
7395        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
7396        assert_eq!(resumed.state.status, runner.state.status);
7397        assert!(
7398            resumed.state.parked,
7399            "still parked until `execute` actually walks the graph again"
7400        );
7401        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
7402    }
7403
7404    /// A fresh open question on `run`, stored and handed back for assertions.
7405    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
7406        let mut q = ask::Question::new(
7407            run.to_owned(),
7408            "implement".to_owned(),
7409            "impl-A".to_owned(),
7410            "Which storage backend should the cache use?".to_owned(),
7411            String::new(),
7412            vec!["SQLite".to_owned(), "Redis".to_owned()],
7413        );
7414        store.put(&mut q).unwrap();
7415        q
7416    }
7417
7418    #[test]
7419    fn a_failed_runs_open_question_is_abandoned() {
7420        ask_test_home();
7421        let store = ask::Questions::open();
7422        let mut runner = runner_at(RunStatus::Failed);
7423        let run = runner.state.id.clone();
7424        let q = ask_open_question(&store, &run);
7425
7426        runner.settle_questions();
7427
7428        let back = store.get(&q.id).unwrap();
7429        assert!(
7430            !back.status.open(),
7431            "the seat that asked died with the run; nobody is left to read an answer"
7432        );
7433        assert!(
7434            back.detail.contains(&run) && back.detail.contains("failed"),
7435            "the reason names what the run became, not just that it is gone: {}",
7436            back.detail
7437        );
7438    }
7439
7440    #[test]
7441    fn a_merged_runs_open_question_is_abandoned_too() {
7442        ask_test_home();
7443        let store = ask::Questions::open();
7444        // A run that finishes cleanly still leaves nobody to read an answer -
7445        // this is not only a failure-path cleanup.
7446        for status in [RunStatus::Merged, RunStatus::Ready] {
7447            let mut runner = runner_at(status);
7448            let run = runner.state.id.clone();
7449            let q = ask_open_question(&store, &run);
7450
7451            runner.settle_questions();
7452
7453            let back = store.get(&q.id).unwrap();
7454            assert!(
7455                !back.status.open(),
7456                "{status:?} run's question must not outlive the run"
7457            );
7458        }
7459    }
7460
7461    #[test]
7462    fn a_still_resumable_runs_open_question_is_left_alone() {
7463        ask_test_home();
7464        let store = ask::Questions::open();
7465        // `Blocked` and `Stalled` can still be resumed — the candidates, the
7466        // review round and the seat sessions are all still on disk — so a
7467        // question asked mid-round may yet get a real answer from a real
7468        // resume. Sweeping it here would be exactly the failure mode this
7469        // whole feature exists to avoid on the other side.
7470        for status in [RunStatus::Blocked, RunStatus::Stalled] {
7471            let mut runner = runner_at(status);
7472            let run = runner.state.id.clone();
7473            let q = ask_open_question(&store, &run);
7474
7475            runner.settle_questions();
7476
7477            let back = store.get(&q.id).unwrap();
7478            assert!(
7479                back.status.open(),
7480                "{status:?} is still alive; the question must still be waiting"
7481            );
7482        }
7483    }
7484
7485    #[test]
7486    fn settle_questions_never_touches_an_already_answered_question() {
7487        ask_test_home();
7488        let store = ask::Questions::open();
7489        let mut runner = runner_at(RunStatus::Failed);
7490        let run = runner.state.id.clone();
7491        let mut q = ask_open_question(&store, &run);
7492        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
7493            .unwrap();
7494        store.put(&mut q).unwrap();
7495
7496        // Called twice, the way a crash-recovered daemon reclaim and the
7497        // graph's own cleanup both can for the same run — `abandon_for_run`
7498        // only ever touches what is still open, so this must be inert both
7499        // times, not merely the second.
7500        runner.settle_questions();
7501        runner.settle_questions();
7502
7503        let back = store.get(&q.id).unwrap();
7504        assert_eq!(
7505            back.status,
7506            ask::QuestionStatus::Answered,
7507            "a real answer is a decision on record, never overwritten by a sweep"
7508        );
7509    }
7510
7511    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
7512    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
7513    /// without merging, whose winner is still the operator's answer to read.
7514    /// Nothing previously called `fold_run` itself with a real `tally`, so
7515    /// this is the first test to pin down the one distinction the whole
7516    /// automatic-fold feature depends on: the winner's worktree and branch
7517    /// must survive, everything else sharing the run's worktree bay - a
7518    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
7519    /// second sweep treats every non-winner directory under the bay alike -
7520    /// must not.
7521    #[tokio::test]
7522    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
7523        crate::run::set_home(std::env::temp_dir().join("magi-graph-fold-run-tests-home"));
7524        let tmp = tempfile::tempdir().expect("tempdir");
7525        let repo = tmp.path().join("repo");
7526        std::fs::create_dir_all(&repo).unwrap();
7527        init_repo(&repo);
7528
7529        let mut config = Config::default();
7530        config.graph.worktree_root = Some(tmp.path().join("wt"));
7531
7532        let mut state = RunState::new(
7533            repo.clone(),
7534            "main".to_owned(),
7535            "deadbeef".to_owned(),
7536            "task".to_owned(),
7537            config,
7538        );
7539        let root = state.worktree_root();
7540        let wt_a = root.join("cand-A");
7541        let wt_b = root.join("cand-B");
7542        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
7543            .await
7544            .expect("worktree A");
7545        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
7546            .await
7547            .expect("worktree B");
7548
7549        state.candidates = vec![
7550            Candidate {
7551                index: 0,
7552                label: 'A',
7553                agent: "alpha".to_owned(),
7554                branch: "magi/x/A".to_owned(),
7555                worktree: wt_a.clone(),
7556                summary: String::new(),
7557                stat: String::new(),
7558                files: 0,
7559                commits: 0,
7560                empty: false,
7561                failed: None,
7562                verified_noop: None,
7563                duration_ms: 0,
7564                folded: false,
7565            },
7566            Candidate {
7567                index: 1,
7568                label: 'B',
7569                agent: "beta".to_owned(),
7570                branch: "magi/x/B".to_owned(),
7571                worktree: wt_b.clone(),
7572                summary: String::new(),
7573                stat: String::new(),
7574                files: 0,
7575                commits: 0,
7576                empty: false,
7577                failed: None,
7578                verified_noop: None,
7579                duration_ms: 0,
7580                folded: false,
7581            },
7582        ];
7583        state.tally = Some(Tally {
7584            first_choice: BTreeMap::from([('A', 1)]),
7585            borda: BTreeMap::new(),
7586            winner: 'A',
7587            rankings: 1,
7588            unanimous_initial: true,
7589            deliberated: false,
7590            changed_votes: 0,
7591            unanimous_final: true,
7592            tie_break: None,
7593            judges: 1,
7594            present: 1,
7595            quorum: 1,
7596            met_quorum: true,
7597            uncontested: None,
7598        });
7599        state.status = RunStatus::Ready;
7600
7601        fold_run(&mut state, false, &crate::run::home())
7602            .await
7603            .expect("fold_run");
7604
7605        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
7606        assert!(
7607            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
7608            "the unmerged winner's branch survives"
7609        );
7610        assert!(
7611            !state.candidates[0].folded,
7612            "the winner is not marked folded"
7613        );
7614
7615        assert!(!wt_b.exists(), "the loser's worktree is removed");
7616        assert!(
7617            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
7618            "the loser's branch is removed"
7619        );
7620        assert!(state.candidates[1].folded, "the loser is marked folded");
7621    }
7622
7623    /// `status == Ready` used to be read as "this is the harmless
7624    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
7625    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
7626    /// run's PR was closed without merging — and reentering `merge` with
7627    /// `mode` still `Pr` does not know the difference, so it pushed and
7628    /// opened a second pull request. `mode == Local` reproduces the same
7629    /// blind spot without a network call: reentry must not attempt another
7630    /// git merge once this node has already recorded an outcome.
7631    #[tokio::test]
7632    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
7633        let tmp = tempfile::tempdir().expect("tempdir");
7634        let repo = tmp.path().join("repo");
7635        std::fs::create_dir_all(&repo).unwrap();
7636        init_repo(&repo);
7637
7638        let mut config = Config::default();
7639        config.merge.mode = MergeMode::Local;
7640
7641        let mut state = RunState::new(
7642            repo.clone(),
7643            "main".to_owned(),
7644            "deadbeef".to_owned(),
7645            "task".to_owned(),
7646            config,
7647        );
7648        state.candidates = vec![Candidate {
7649            index: 0,
7650            label: 'A',
7651            agent: "alpha".to_owned(),
7652            branch: "does-not-exist".to_owned(),
7653            worktree: repo.clone(),
7654            summary: String::new(),
7655            stat: String::new(),
7656            files: 0,
7657            commits: 0,
7658            empty: false,
7659            failed: None,
7660            verified_noop: None,
7661            duration_ms: 0,
7662            folded: false,
7663        }];
7664        state.tally = Some(Tally {
7665            first_choice: BTreeMap::from([('A', 1)]),
7666            borda: BTreeMap::new(),
7667            winner: 'A',
7668            rankings: 1,
7669            unanimous_initial: true,
7670            deliberated: false,
7671            changed_votes: 0,
7672            unanimous_final: true,
7673            tie_break: None,
7674            judges: 0,
7675            present: 0,
7676            quorum: 0,
7677            met_quorum: true,
7678            uncontested: Some("only candidate A produced a change".to_owned()),
7679        });
7680        state.reviews = vec![ReviewRound {
7681            round: 1,
7682            head: "deadbeef".to_owned(),
7683            verified_head: None,
7684            verified_at: None,
7685            reviews: Vec::new(),
7686            e2e: Vec::new(),
7687            fix: None,
7688            blocking: 0,
7689            answered: 0,
7690            expected: 0,
7691            clean: true,
7692            verify_retried: false,
7693            e2e_deferred: false,
7694            e2e_defer_reason: None,
7695            progressed: false,
7696            vote_split: false,
7697            reconsideration: Vec::new(),
7698            verdict: None,
7699        }];
7700        state.gate = vec![CommandOutcome {
7701            command: "test".to_owned(),
7702            code: Some(0),
7703            output_tail: String::new(),
7704            duration_ms: 0,
7705            resource_blocked: false,
7706        }];
7707        state.gate_ran = true;
7708        // Reached its conclusion already — e.g. `land` closing the PR without
7709        // merging it, which (like the honest `MergeMode::None` path) leaves
7710        // `status` at `Ready`. The recorded outcome is what actually marks
7711        // this node done.
7712        state.status = RunStatus::Ready;
7713        state.merge = Some(MergeOutcome {
7714            mode: MergeMode::Local,
7715            ok: false,
7716            detail: "already concluded".to_owned(),
7717        });
7718
7719        let mut runner = Runner {
7720            state,
7721            roles: ResolvedRoles {
7722                implementers: Vec::new(),
7723                judges: Vec::new(),
7724                reviewers: Vec::new(),
7725                fixer: None,
7726                conductor: conductor(),
7727                implementer_roster: Vec::new(),
7728            },
7729            sem: Arc::new(Semaphore::new(1)),
7730            pause: Pause::new(),
7731            interrupt: Pause::new(),
7732        };
7733
7734        runner.merge().await.expect("merge");
7735
7736        assert_eq!(
7737            runner.state.status,
7738            RunStatus::Ready,
7739            "a concluded run's status must not change on reentry"
7740        );
7741        assert_eq!(
7742            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
7743            Some("already concluded"),
7744            "merge must not run again once the node already recorded an outcome"
7745        );
7746    }
7747
7748    /// `gate` leaves `state.gate_ran` false both before it has ever run and
7749    /// when its last attempt was resource-blocked (the shared build cache
7750    /// could not be acquired or confirmed fresh in time - see
7751    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
7752    /// `Vec` this also leaves behind used to read as "nothing failed" and let
7753    /// a run merge a tree the gate never actually checked - exactly the case
7754    /// a contended cache produces on every retry until it clears. `merge`
7755    /// must refuse until `gate` has actually recorded an attempt.
7756    #[tokio::test]
7757    async fn merge_refuses_a_gate_that_has_not_actually_run() {
7758        let tmp = tempfile::tempdir().expect("tempdir");
7759        let repo = tmp.path().join("repo");
7760        std::fs::create_dir_all(&repo).unwrap();
7761        init_repo(&repo);
7762
7763        let mut config = Config::default();
7764        config.merge.mode = MergeMode::Local;
7765
7766        let mut state = RunState::new(
7767            repo.clone(),
7768            "main".to_owned(),
7769            "deadbeef".to_owned(),
7770            "task".to_owned(),
7771            config,
7772        );
7773        state.candidates = vec![Candidate {
7774            index: 0,
7775            label: 'A',
7776            agent: "alpha".to_owned(),
7777            branch: "does-not-exist".to_owned(),
7778            worktree: repo.clone(),
7779            summary: String::new(),
7780            stat: String::new(),
7781            files: 0,
7782            commits: 0,
7783            empty: false,
7784            failed: None,
7785            verified_noop: None,
7786            duration_ms: 0,
7787            folded: false,
7788        }];
7789        state.tally = Some(Tally {
7790            first_choice: BTreeMap::from([('A', 1)]),
7791            borda: BTreeMap::new(),
7792            winner: 'A',
7793            rankings: 1,
7794            unanimous_initial: true,
7795            deliberated: false,
7796            changed_votes: 0,
7797            unanimous_final: true,
7798            tie_break: None,
7799            judges: 0,
7800            present: 0,
7801            quorum: 0,
7802            met_quorum: true,
7803            uncontested: Some("only candidate A produced a change".to_owned()),
7804        });
7805        state.reviews = vec![ReviewRound {
7806            round: 1,
7807            head: "deadbeef".to_owned(),
7808            verified_head: None,
7809            verified_at: None,
7810            reviews: Vec::new(),
7811            e2e: Vec::new(),
7812            fix: None,
7813            blocking: 0,
7814            answered: 0,
7815            expected: 0,
7816            clean: true,
7817            verify_retried: false,
7818            e2e_deferred: false,
7819            e2e_defer_reason: None,
7820            progressed: false,
7821            vote_split: false,
7822            reconsideration: Vec::new(),
7823            verdict: None,
7824        }];
7825        // The point: `gate` has not recorded anything yet.
7826        state.gate = Vec::new();
7827        state.gate_ran = false;
7828        state.status = RunStatus::Gating;
7829
7830        let mut runner = Runner {
7831            state,
7832            roles: ResolvedRoles {
7833                implementers: Vec::new(),
7834                judges: Vec::new(),
7835                reviewers: Vec::new(),
7836                fixer: None,
7837                conductor: conductor(),
7838                implementer_roster: Vec::new(),
7839            },
7840            sem: Arc::new(Semaphore::new(1)),
7841            pause: Pause::new(),
7842            interrupt: Pause::new(),
7843        };
7844
7845        runner.merge().await.expect("merge");
7846
7847        assert!(
7848            runner.state.merge.is_none(),
7849            "an empty gate must never be read as a passing one: {:?}",
7850            runner.state.merge
7851        );
7852    }
7853
7854    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
7855    /// commands configured and `merge.mode` is `none` (a review-only run).
7856    /// `gate` must still record a real attempt — zero commands, vacuously
7857    /// passed — rather than leaving `state.gate` empty in a way `merge`
7858    /// cannot tell apart from "never ran"; otherwise the run reaches
7859    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
7860    #[tokio::test]
7861    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
7862        let tmp = tempfile::tempdir().expect("tempdir");
7863        let repo = tmp.path().join("repo");
7864        std::fs::create_dir_all(&repo).unwrap();
7865        init_repo(&repo);
7866
7867        // Default config: `verify.gate` empty, `merge.mode` is `none`.
7868        let config = Config::default();
7869
7870        let mut state = RunState::new(
7871            repo.clone(),
7872            "main".to_owned(),
7873            "deadbeef".to_owned(),
7874            "task".to_owned(),
7875            config,
7876        );
7877        state.candidates = vec![Candidate {
7878            index: 0,
7879            label: 'A',
7880            agent: "alpha".to_owned(),
7881            branch: "does-not-exist".to_owned(),
7882            worktree: repo.clone(),
7883            summary: String::new(),
7884            stat: String::new(),
7885            files: 0,
7886            commits: 0,
7887            empty: false,
7888            failed: None,
7889            verified_noop: None,
7890            duration_ms: 0,
7891            folded: false,
7892        }];
7893        state.tally = Some(Tally {
7894            first_choice: BTreeMap::from([('A', 1)]),
7895            borda: BTreeMap::new(),
7896            winner: 'A',
7897            rankings: 1,
7898            unanimous_initial: true,
7899            deliberated: false,
7900            changed_votes: 0,
7901            unanimous_final: true,
7902            tie_break: None,
7903            judges: 0,
7904            present: 0,
7905            quorum: 0,
7906            met_quorum: true,
7907            uncontested: Some("only candidate A produced a change".to_owned()),
7908        });
7909        state.reviews = vec![ReviewRound {
7910            round: 1,
7911            head: "deadbeef".to_owned(),
7912            verified_head: None,
7913            verified_at: None,
7914            reviews: Vec::new(),
7915            e2e: Vec::new(),
7916            fix: None,
7917            blocking: 0,
7918            answered: 0,
7919            expected: 0,
7920            clean: true,
7921            verify_retried: false,
7922            e2e_deferred: false,
7923            e2e_defer_reason: None,
7924            progressed: false,
7925            vote_split: false,
7926            reconsideration: Vec::new(),
7927            verdict: None,
7928        }];
7929
7930        let mut runner = Runner {
7931            state,
7932            roles: ResolvedRoles {
7933                implementers: Vec::new(),
7934                judges: Vec::new(),
7935                reviewers: Vec::new(),
7936                fixer: None,
7937                conductor: conductor(),
7938                implementer_roster: Vec::new(),
7939            },
7940            sem: Arc::new(Semaphore::new(1)),
7941            pause: Pause::new(),
7942            interrupt: Pause::new(),
7943        };
7944
7945        runner.gate().await.expect("gate");
7946        assert!(
7947            runner.state.gate_ran,
7948            "zero configured commands is still a real attempt, not an unrun gate"
7949        );
7950        assert!(runner.state.gate.is_empty());
7951        assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
7952        assert_ne!(
7953            runner.state.status,
7954            RunStatus::Blocked,
7955            "a gate with nothing to check must not read as failed"
7956        );
7957
7958        runner.merge().await.expect("merge");
7959        assert_eq!(
7960            runner.state.status,
7961            RunStatus::Ready,
7962            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
7963        );
7964    }
7965
7966    /// `Config::cache_dir` is derived from `verify.e2e` as well as
7967    /// `verify.gate` (so the e2e leg and the final gate never build against
7968    /// different directories). With zero `verify.gate` commands but a
7969    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
7970    /// that lease before discovering it had nothing to run - so a repo with
7971    /// no gate commands could come back `resource_blocked` (and therefore
7972    /// still `gate_ran == false`) on nothing but an unrelated run holding the
7973    /// cache, exactly the contention this run's own zero commands could
7974    /// never have touched. `gate` must recognise there is nothing to check
7975    /// before it ever asks for the lease.
7976    #[tokio::test]
7977    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
7978        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
7979        let home = crate::run::home();
7980
7981        let tmp = tempfile::tempdir().expect("tempdir");
7982        let repo = tmp.path().join("repo");
7983        std::fs::create_dir_all(&repo).unwrap();
7984        init_repo(&repo);
7985        // Unique to this test, so holding its lease cannot collide with
7986        // another test sharing the same process-wide `home`.
7987        let cache_dir = tmp.path().join("target");
7988
7989        let mut config = Config::default();
7990        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
7991        // `verify.gate` stays empty (the default). Bounded so a regression
7992        // that does start waiting fails the test in seconds, not hangs it.
7993        config.graph.timeout_verify = Some(2);
7994
7995        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
7996        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
7997            .expect("no io error acquiring directly")
7998        {
7999            crate::cache::AcquireOutcome::Acquired(g) => g,
8000            crate::cache::AcquireOutcome::Busy(b) => {
8001                panic!("expected the direct acquire to win the lease first: {b:?}")
8002            }
8003        };
8004
8005        let mut state = RunState::new(
8006            repo.clone(),
8007            "main".to_owned(),
8008            "deadbeef".to_owned(),
8009            "task".to_owned(),
8010            config,
8011        );
8012        state.candidates = vec![Candidate {
8013            index: 0,
8014            label: 'A',
8015            agent: "alpha".to_owned(),
8016            branch: "does-not-exist".to_owned(),
8017            worktree: repo.clone(),
8018            summary: String::new(),
8019            stat: String::new(),
8020            files: 0,
8021            commits: 0,
8022            empty: false,
8023            failed: None,
8024            verified_noop: None,
8025            duration_ms: 0,
8026            folded: false,
8027        }];
8028        state.tally = Some(Tally {
8029            first_choice: BTreeMap::from([('A', 1)]),
8030            borda: BTreeMap::new(),
8031            winner: 'A',
8032            rankings: 1,
8033            unanimous_initial: true,
8034            deliberated: false,
8035            changed_votes: 0,
8036            unanimous_final: true,
8037            tie_break: None,
8038            judges: 0,
8039            present: 0,
8040            quorum: 0,
8041            met_quorum: true,
8042            uncontested: Some("only candidate A produced a change".to_owned()),
8043        });
8044        state.reviews = vec![ReviewRound {
8045            round: 1,
8046            head: "deadbeef".to_owned(),
8047            verified_head: None,
8048            verified_at: None,
8049            reviews: Vec::new(),
8050            e2e: Vec::new(),
8051            fix: None,
8052            blocking: 0,
8053            answered: 0,
8054            expected: 0,
8055            clean: true,
8056            verify_retried: false,
8057            e2e_deferred: false,
8058            e2e_defer_reason: None,
8059            progressed: false,
8060            vote_split: false,
8061            reconsideration: Vec::new(),
8062            verdict: None,
8063        }];
8064
8065        let mut runner = Runner {
8066            state,
8067            roles: ResolvedRoles {
8068                implementers: Vec::new(),
8069                judges: Vec::new(),
8070                reviewers: Vec::new(),
8071                fixer: None,
8072                conductor: conductor(),
8073                implementer_roster: Vec::new(),
8074            },
8075            sem: Arc::new(Semaphore::new(1)),
8076            pause: Pause::new(),
8077            interrupt: Pause::new(),
8078        };
8079
8080        let started = std::time::Instant::now();
8081        runner.gate().await.expect("gate");
8082        assert!(
8083            started.elapsed() < Duration::from_secs(1),
8084            "a gate with nothing to run must never wait on a lease it never needed"
8085        );
8086        assert!(
8087            runner.state.gate_ran,
8088            "zero commands is still a real, immediate attempt"
8089        );
8090        assert!(runner.state.gate.is_empty());
8091        assert_ne!(
8092            runner.state.status,
8093            RunStatus::Blocked,
8094            "must not read as resource-blocked on a lease it never asked for"
8095        );
8096    }
8097
8098    /// The addendum's second gap: a `verify.gate` command running for real
8099    /// wall-clock time had nothing at all to show for it in `active` before
8100    /// `run_commands` learned to record it — a run could sit in `Gating` for
8101    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
8102    /// what was actually happening. Proven with a genuinely still-running
8103    /// command, not just a before/after check on the final state: a poller
8104    /// task reads the same `run.json` `gate()` is writing, the same way the
8105    /// phone or `magi show` would, while the shell command is still blocked
8106    /// on its own release marker.
8107    #[tokio::test]
8108    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
8109        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8110
8111        let tmp = tempfile::tempdir().expect("tempdir");
8112        let repo = tmp.path().join("repo");
8113        std::fs::create_dir_all(&repo).unwrap();
8114        init_repo(&repo);
8115
8116        let mut config = Config::default();
8117        config.verify.gate = vec![
8118            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
8119             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
8120                .to_owned(),
8121        ];
8122
8123        let mut state = RunState::new(
8124            repo.clone(),
8125            "main".to_owned(),
8126            "deadbeef".to_owned(),
8127            "task".to_owned(),
8128            config,
8129        );
8130        let run_id = state.id.clone();
8131        state.candidates = vec![Candidate {
8132            index: 0,
8133            label: 'A',
8134            agent: "alpha".to_owned(),
8135            branch: "does-not-exist".to_owned(),
8136            worktree: repo.clone(),
8137            summary: String::new(),
8138            stat: String::new(),
8139            files: 0,
8140            commits: 0,
8141            empty: false,
8142            failed: None,
8143            verified_noop: None,
8144            duration_ms: 0,
8145            folded: false,
8146        }];
8147        state.tally = Some(Tally {
8148            first_choice: BTreeMap::from([('A', 1)]),
8149            borda: BTreeMap::new(),
8150            winner: 'A',
8151            rankings: 1,
8152            unanimous_initial: true,
8153            deliberated: false,
8154            changed_votes: 0,
8155            unanimous_final: true,
8156            tie_break: None,
8157            judges: 0,
8158            present: 0,
8159            quorum: 0,
8160            met_quorum: true,
8161            uncontested: Some("only candidate A produced a change".to_owned()),
8162        });
8163        state.reviews = vec![ReviewRound {
8164            round: 1,
8165            head: "deadbeef".to_owned(),
8166            verified_head: None,
8167            verified_at: None,
8168            reviews: Vec::new(),
8169            e2e: Vec::new(),
8170            fix: None,
8171            blocking: 0,
8172            answered: 0,
8173            expected: 0,
8174            clean: true,
8175            verify_retried: false,
8176            e2e_deferred: false,
8177            e2e_defer_reason: None,
8178            progressed: false,
8179            vote_split: false,
8180            reconsideration: Vec::new(),
8181            verdict: None,
8182        }];
8183
8184        let mut runner = Runner {
8185            state,
8186            roles: ResolvedRoles {
8187                implementers: Vec::new(),
8188                judges: Vec::new(),
8189                reviewers: Vec::new(),
8190                fixer: None,
8191                conductor: conductor(),
8192                implementer_roster: Vec::new(),
8193            },
8194            sem: Arc::new(Semaphore::new(1)),
8195            pause: Pause::new(),
8196            interrupt: Pause::new(),
8197        };
8198
8199        let started_marker = repo.join("started.marker");
8200        let release_marker = repo.join("release.marker");
8201        let poller = tokio::spawn(async move {
8202            // Bounded so a regression that never records the task entry
8203            // fails this test in seconds instead of hanging the suite —
8204            // the same shape `a_park_requested_while_a_seat_is_mid_call_
8205            // does_not_cut_it_short` uses for the same reason.
8206            for _ in 0..100 {
8207                if started_marker.exists()
8208                    && let Ok(s) = crate::run::RunState::load(&run_id)
8209                    && let Some(a) = s.active.get("gate")
8210                {
8211                    std::fs::write(&release_marker, b"go").expect("release marker");
8212                    return Some(a.clone());
8213                }
8214                tokio::time::sleep(Duration::from_millis(50)).await;
8215            }
8216            None
8217        });
8218
8219        runner.gate().await.expect("gate");
8220        let captured = poller.await.expect("poller task");
8221        let captured = captured.expect(
8222            "the poller never saw a `gate` task entry in run.json while the command was \
8223             still blocked on its own release marker",
8224        );
8225
8226        assert_eq!(captured.task.as_deref(), Some("gate"));
8227        assert_eq!(captured.node, "gate");
8228        assert_eq!(captured.index, Some(1));
8229        assert_eq!(captured.total, Some(1));
8230        assert!(
8231            captured
8232                .command
8233                .as_deref()
8234                .is_some_and(|c| c.contains("started.marker")),
8235            "{captured:?}"
8236        );
8237
8238        assert!(
8239            runner.state.active.is_empty(),
8240            "the entry must be cleared once the command actually finished: {:?}",
8241            runner.state.active
8242        );
8243        assert!(runner.state.gate_ran);
8244        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
8245    }
8246
8247    /// The shape the incident this whole fix responds to actually had: the
8248    /// round budget spent, the last round's own e2e blocked on the shared
8249    /// build cache (held here by a live pid — this test process — exactly
8250    /// `cache`'s own unit tests' pattern for "another owner, still alive"
8251    /// without forking a process). `stop_reviewing` must retry it — not
8252    /// silently leave the round looking untouched (the catch-up-only half of
8253    /// the bug), and not read the contention as a red `e2e` and block the
8254    /// run on it (the other half). Called directly, the same way
8255    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
8256    /// above exercises `gate`, so this never needs a real cargo build to
8257    /// reach: the lease is never released, so `with_cache_lease` never gets
8258    /// past acquiring it into anything that would need a real workspace.
8259    #[tokio::test]
8260    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
8261        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8262        let home = crate::run::home();
8263
8264        let tmp = tempfile::tempdir().expect("tempdir");
8265        let repo = tmp.path().join("repo");
8266        std::fs::create_dir_all(&repo).unwrap();
8267        init_repo(&repo);
8268        let head = crate::git::rev_parse(&repo, "HEAD")
8269            .await
8270            .expect("rev-parse");
8271        // Unique to this test, so holding its lease cannot collide with
8272        // another test sharing the same process-wide `home`.
8273        let cache_dir = tmp.path().join("target");
8274
8275        let mut config = Config::default();
8276        config.verify.e2e = vec![format!(
8277            "CARGO_TARGET_DIR='{}' test -f README.md",
8278            cache_dir.display()
8279        )];
8280        config.graph.review_rounds = 1;
8281        // Bounded so a regression that does start waiting fails the test in
8282        // seconds, not hangs it.
8283        config.graph.timeout_verify = Some(2);
8284
8285        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
8286        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
8287            .expect("no io error acquiring directly")
8288        {
8289            crate::cache::AcquireOutcome::Acquired(g) => g,
8290            crate::cache::AcquireOutcome::Busy(b) => {
8291                panic!("expected the direct acquire to win the lease first: {b:?}")
8292            }
8293        };
8294
8295        let mut state = RunState::new(
8296            repo.clone(),
8297            "main".to_owned(),
8298            head.clone(),
8299            "task".to_owned(),
8300            config,
8301        );
8302        state.candidates = vec![Candidate {
8303            index: 0,
8304            label: 'A',
8305            agent: "alpha".to_owned(),
8306            branch: "does-not-exist".to_owned(),
8307            worktree: repo.clone(),
8308            summary: String::new(),
8309            stat: String::new(),
8310            files: 0,
8311            commits: 0,
8312            empty: false,
8313            failed: None,
8314            verified_noop: None,
8315            duration_ms: 0,
8316            folded: false,
8317        }];
8318        state.tally = Some(Tally {
8319            first_choice: BTreeMap::from([('A', 1)]),
8320            borda: BTreeMap::new(),
8321            winner: 'A',
8322            rankings: 1,
8323            unanimous_initial: true,
8324            deliberated: false,
8325            changed_votes: 0,
8326            unanimous_final: true,
8327            tie_break: None,
8328            judges: 0,
8329            present: 0,
8330            quorum: 0,
8331            met_quorum: true,
8332            uncontested: Some("only candidate A produced a change".to_owned()),
8333        });
8334        // The round budget's last round, deferred: `needs_catchup_run`'s
8335        // other trigger. `stop_reviewing`'s retry machinery must treat this
8336        // exactly like a resource-blocked attempt once it actually runs.
8337        state.reviews = vec![ReviewRound {
8338            round: 1,
8339            head: head.clone(),
8340            verified_head: None,
8341            verified_at: None,
8342            reviews: Vec::new(),
8343            e2e: Vec::new(),
8344            fix: None,
8345            blocking: 1,
8346            answered: 1,
8347            expected: 1,
8348            clean: false,
8349            verify_retried: false,
8350            e2e_deferred: true,
8351            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
8352            progressed: false,
8353            vote_split: false,
8354            reconsideration: Vec::new(),
8355            verdict: None,
8356        }];
8357
8358        let mut runner = Runner {
8359            state,
8360            roles: ResolvedRoles {
8361                implementers: Vec::new(),
8362                judges: Vec::new(),
8363                reviewers: Vec::new(),
8364                fixer: None,
8365                conductor: conductor(),
8366                implementer_roster: Vec::new(),
8367            },
8368            sem: Arc::new(Semaphore::new(1)),
8369            pause: Pause::new(),
8370            interrupt: Pause::new(),
8371        };
8372
8373        let shell = runner.state.config.shell();
8374        runner
8375            .stop_reviewing("round budget spent", &shell, &repo)
8376            .await
8377            .expect("stop_reviewing");
8378
8379        let last = runner.state.reviews.last().expect("round record");
8380        assert_eq!(
8381            last.e2e_status(),
8382            E2eStatus::ResourceBlocked,
8383            "the shared cache is still held; the attempt must read as blocked, not deferred or \
8384             failed: {last:?}"
8385        );
8386        assert_eq!(
8387            last.verified_head.as_deref(),
8388            Some(head.as_str()),
8389            "which commit this attempt targeted is known even though nothing finished checking \
8390             it"
8391        );
8392        let first_attempt_at = last
8393            .verified_at
8394            .expect("when this attempt ran is known too");
8395        assert_ne!(
8396            runner.state.status,
8397            RunStatus::Blocked,
8398            "contention is evidence about the machine, not the patch — it must not settle the \
8399             run as blocked: {:?}",
8400            runner.state.status
8401        );
8402        assert!(
8403            !runner
8404                .state
8405                .events
8406                .iter()
8407                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
8408            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
8409            runner.state.events
8410        );
8411
8412        // The cache is still held: a later reentry must retry the same
8413        // round's verification again — not leave it looking exactly as
8414        // untouched as the first blocked attempt, which is indistinguishable
8415        // from never having tried again at all.
8416        runner
8417            .stop_reviewing("round budget spent", &shell, &repo)
8418            .await
8419            .expect("stop_reviewing retry");
8420        assert_eq!(
8421            runner.state.reviews.len(),
8422            1,
8423            "no new round was started: {:?}",
8424            runner.state.reviews
8425        );
8426        let last = runner.state.reviews.last().expect("round record");
8427        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
8428        assert!(
8429            last.verified_at.expect("still known") > first_attempt_at,
8430            "a second reentry must be a fresh attempt, not a stale copy of the first"
8431        );
8432        assert_ne!(runner.state.status, RunStatus::Blocked);
8433
8434        held.release();
8435    }
8436
8437    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
8438    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
8439    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
8440    /// own top-of-function fast path (`review_conclusion`) correctly reads
8441    /// this shape as `None` rather than guessing `Blocked`, and the loop's
8442    /// own `for` range is empty once the round budget is spent, so
8443    /// `review_loop` must retry the check itself rather than silently doing
8444    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
8445    /// above exercises directly, but through `review_loop`'s own entry point
8446    /// this time, proving the wiring between the two rather than just the
8447    /// retry logic in isolation.
8448    #[tokio::test]
8449    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
8450        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8451        let home = crate::run::home();
8452
8453        let tmp = tempfile::tempdir().expect("tempdir");
8454        let repo = tmp.path().join("repo");
8455        std::fs::create_dir_all(&repo).unwrap();
8456        init_repo(&repo);
8457        let head = crate::git::rev_parse(&repo, "HEAD")
8458            .await
8459            .expect("rev-parse");
8460        let cache_dir = tmp.path().join("target");
8461
8462        let mut config = Config::default();
8463        config.verify.e2e = vec![format!(
8464            "CARGO_TARGET_DIR='{}' test -f README.md",
8465            cache_dir.display()
8466        )];
8467        config.graph.review_rounds = 1;
8468        config.graph.timeout_verify = Some(2);
8469
8470        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
8471        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
8472            .expect("no io error acquiring directly")
8473        {
8474            crate::cache::AcquireOutcome::Acquired(g) => g,
8475            crate::cache::AcquireOutcome::Busy(b) => {
8476                panic!("expected the direct acquire to win the lease first: {b:?}")
8477            }
8478        };
8479
8480        let mut state = RunState::new(
8481            repo.clone(),
8482            "main".to_owned(),
8483            head.clone(),
8484            "task".to_owned(),
8485            config,
8486        );
8487        state.candidates = vec![Candidate {
8488            index: 0,
8489            label: 'A',
8490            agent: "alpha".to_owned(),
8491            branch: "does-not-exist".to_owned(),
8492            worktree: repo.clone(),
8493            summary: String::new(),
8494            stat: String::new(),
8495            files: 0,
8496            commits: 0,
8497            empty: false,
8498            failed: None,
8499            verified_noop: None,
8500            duration_ms: 0,
8501            folded: false,
8502        }];
8503        state.tally = Some(Tally {
8504            first_choice: BTreeMap::from([('A', 1)]),
8505            borda: BTreeMap::new(),
8506            winner: 'A',
8507            rankings: 1,
8508            unanimous_initial: true,
8509            deliberated: false,
8510            changed_votes: 0,
8511            unanimous_final: true,
8512            tie_break: None,
8513            judges: 0,
8514            present: 0,
8515            quorum: 0,
8516            met_quorum: true,
8517            uncontested: Some("only candidate A produced a change".to_owned()),
8518        });
8519        // The exact shape a prior process's `stop_reviewing` would have left
8520        // on disk: the round budget's last round, a real attempt already
8521        // made and already resource-blocked.
8522        state.reviews = vec![ReviewRound {
8523            round: 1,
8524            head: head.clone(),
8525            verified_head: Some(head.clone()),
8526            verified_at: Some(jiff::Timestamp::now()),
8527            reviews: Vec::new(),
8528            e2e: vec![CommandOutcome {
8529                command: format!(
8530                    "CARGO_TARGET_DIR='{}' test -f README.md",
8531                    cache_dir.display()
8532                ),
8533                code: None,
8534                output_tail: "waiting for the shared build cache".to_owned(),
8535                duration_ms: 0,
8536                resource_blocked: true,
8537            }],
8538            fix: None,
8539            blocking: 1,
8540            answered: 1,
8541            expected: 1,
8542            clean: false,
8543            verify_retried: false,
8544            e2e_deferred: false,
8545            e2e_defer_reason: None,
8546            progressed: false,
8547            vote_split: false,
8548            reconsideration: Vec::new(),
8549            verdict: None,
8550        }];
8551
8552        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
8553        let mut runner = Runner {
8554            state,
8555            roles: ResolvedRoles {
8556                implementers: Vec::new(),
8557                judges: Vec::new(),
8558                reviewers: Vec::new(),
8559                fixer: None,
8560                conductor: conductor(),
8561                implementer_roster: Vec::new(),
8562            },
8563            sem: Arc::new(Semaphore::new(1)),
8564            pause: Pause::new(),
8565            interrupt: Pause::new(),
8566        };
8567
8568        // The lease is still held throughout, so this reentry's own retry is
8569        // also contended — proving `review_loop` actually tried again (not
8570        // that it happened to succeed) is what the timestamp comparison
8571        // below is for.
8572        runner.review_loop().await.expect("review_loop");
8573
8574        assert_eq!(
8575            runner.state.reviews.len(),
8576            1,
8577            "no new round was started on top of the unresolved one: {:?}",
8578            runner.state.reviews
8579        );
8580        let last = &runner.state.reviews[0];
8581        assert_eq!(
8582            last.e2e_status(),
8583            E2eStatus::ResourceBlocked,
8584            "still contended: {last:?}"
8585        );
8586        assert!(
8587            last.verified_at.expect("still known") > first_attempt_at,
8588            "review_loop must have actually retried the check, not left it exactly as found"
8589        );
8590        assert_ne!(
8591            runner.state.status,
8592            RunStatus::Blocked,
8593            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
8594            runner.state.status
8595        );
8596
8597        held.release();
8598    }
8599
8600    #[tokio::test]
8601    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
8602        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8603        let tmp = tempfile::tempdir().expect("tempdir");
8604        let repo = tmp.path().join("repo");
8605        std::fs::create_dir_all(&repo).unwrap();
8606        init_repo(&repo);
8607
8608        let mut config = Config::default();
8609        config.merge.mode = MergeMode::Pr;
8610        config.graph.land = true;
8611        config.graph.land_approval = false;
8612
8613        let mut state = RunState::new(
8614            repo.clone(),
8615            "main".to_owned(),
8616            "deadbeef".to_owned(),
8617            "task".to_owned(),
8618            config,
8619        );
8620        state.candidates = vec![Candidate {
8621            index: 0,
8622            label: 'A',
8623            agent: "alpha".to_owned(),
8624            branch: "does-not-exist".to_owned(),
8625            worktree: repo.clone(),
8626            summary: String::new(),
8627            stat: String::new(),
8628            files: 0,
8629            commits: 0,
8630            empty: false,
8631            failed: None,
8632            verified_noop: None,
8633            duration_ms: 0,
8634            folded: false,
8635        }];
8636        state.tally = Some(Tally {
8637            first_choice: BTreeMap::from([('A', 1)]),
8638            borda: BTreeMap::new(),
8639            winner: 'A',
8640            rankings: 1,
8641            unanimous_initial: true,
8642            deliberated: false,
8643            changed_votes: 0,
8644            unanimous_final: true,
8645            tie_break: None,
8646            judges: 0,
8647            present: 0,
8648            quorum: 0,
8649            met_quorum: true,
8650            uncontested: Some("only candidate A produced a change".to_owned()),
8651        });
8652        state.reviews = vec![ReviewRound {
8653            round: 1,
8654            head: "deadbeef".to_owned(),
8655            verified_head: None,
8656            verified_at: None,
8657            reviews: Vec::new(),
8658            e2e: Vec::new(),
8659            fix: None,
8660            blocking: 0,
8661            answered: 0,
8662            expected: 0,
8663            clean: true,
8664            verify_retried: false,
8665            e2e_deferred: false,
8666            e2e_defer_reason: None,
8667            progressed: false,
8668            vote_split: false,
8669            reconsideration: Vec::new(),
8670            verdict: None,
8671        }];
8672        state.gate = vec![CommandOutcome {
8673            command: "test".to_owned(),
8674            code: Some(0),
8675            output_tail: String::new(),
8676            duration_ms: 0,
8677            resource_blocked: false,
8678        }];
8679        state.gate_ran = true;
8680        // A first pass through `merge` already pushed and opened this pull
8681        // request; `status` is `Landing` because a previous call into `land`
8682        // parked or was interrupted before it reached a terminal outcome.
8683        state.status = RunStatus::Landing;
8684        state.merge = Some(MergeOutcome {
8685            mode: MergeMode::Pr,
8686            ok: true,
8687            detail: "https://example.invalid/x/y/pull/1".to_owned(),
8688        });
8689
8690        // The Landing-resume shortcut calls `run_land` directly rather than
8691        // through `merge`, which is exactly the call site that used to skip
8692        // `settle_questions` - see the fixture below.
8693        ask_test_home();
8694        let store = ask::Questions::open();
8695        let q = ask_open_question(&store, &state.id);
8696
8697        let mut runner = Runner {
8698            state,
8699            roles: ResolvedRoles {
8700                implementers: Vec::new(),
8701                judges: Vec::new(),
8702                reviewers: Vec::new(),
8703                fixer: None,
8704                conductor: conductor(),
8705                implementer_roster: Vec::new(),
8706            },
8707            sem: Arc::new(Semaphore::new(1)),
8708            pause: Pause::new(),
8709            interrupt: Pause::new(),
8710        };
8711
8712        // `execute`, not `merge` directly: the Landing-resume shortcut lives
8713        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
8714        // exactly because `review_loop` would otherwise clobber the marker
8715        // first.
8716        runner.execute().await.expect("execute");
8717
8718        assert_eq!(
8719            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
8720            Some("https://example.invalid/x/y/pull/1"),
8721            "reentry must not push again or open a second pull request over the \
8722             one `land` is already watching"
8723        );
8724        assert_ne!(
8725            runner.state.status,
8726            RunStatus::Landing,
8727            "land could not actually reach the fake pull request, so it must \
8728             have given up rather than left the run silently parked forever"
8729        );
8730        // `land` could not reach the fake pull request, so it gave up into
8731        // `Blocked` - still resumable, so the question must not have been
8732        // swept just because this branch now also calls `settle_questions`.
8733        assert_eq!(runner.state.status, RunStatus::Blocked);
8734        assert!(
8735            store.get(&q.id).unwrap().status.open(),
8736            "Blocked is still alive; settle_questions must have been a no-op here"
8737        );
8738    }
8739
8740    fn state_with_round(round: ReviewRound) -> RunState {
8741        let mut s = RunState::new(
8742            PathBuf::from("/repo"),
8743            "main".to_owned(),
8744            "abc1234".to_owned(),
8745            "add retries".to_owned(),
8746            Config::default(),
8747        );
8748        s.reviews = vec![round];
8749        s
8750    }
8751
8752    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
8753        crate::verdict::Finding {
8754            id: id.to_owned(),
8755            severity,
8756            file: None,
8757            line: None,
8758            title: title.to_owned(),
8759            detail: String::new(),
8760        }
8761    }
8762
8763    #[test]
8764    fn pr_body_names_open_findings_and_declined_ones() {
8765        let round = ReviewRound {
8766            round: 2,
8767            head: "deadbee".to_owned(),
8768            verified_head: None,
8769            verified_at: None,
8770            reviews: vec![ReviewRecord {
8771                attempts: 0,
8772                reviewer: 1,
8773                agent: "alpha".to_owned(),
8774                summary: String::new(),
8775                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
8776                vote: None,
8777                failed: None,
8778                duration_ms: 0,
8779            }],
8780            e2e: vec![CommandOutcome {
8781                command: "cargo test".to_owned(),
8782                code: Some(0),
8783                output_tail: String::new(),
8784                duration_ms: 0,
8785                resource_blocked: false,
8786            }],
8787            verify_retried: false,
8788            e2e_deferred: false,
8789            e2e_defer_reason: None,
8790            fix: Some(FixRecord {
8791                agent: "alpha".to_owned(),
8792                addressed: Vec::new(),
8793                rejected: vec![crate::verdict::Rejection {
8794                    id: "R1-1-1".to_owned(),
8795                    why: "not reachable from any caller".to_owned(),
8796                }],
8797                notes: String::new(),
8798                committed: true,
8799                failed: None,
8800                duration_ms: 0,
8801                continuation: None,
8802            }),
8803            blocking: 0,
8804            answered: 1,
8805            expected: 1,
8806            clean: false,
8807            progressed: true,
8808            vote_split: false,
8809            reconsideration: Vec::new(),
8810            verdict: None,
8811        };
8812        let state = state_with_round(round);
8813        let body = pr_message(&state, 'A').body;
8814
8815        assert!(body.contains("add retries"), "the task must still be there");
8816        assert!(body.contains("R2-1-1"), "{body}");
8817        assert!(body.contains("unused import"), "{body}");
8818        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
8819        assert!(
8820            body.contains("not reachable from any caller"),
8821            "the reason it was declined: {body}"
8822        );
8823    }
8824
8825    #[test]
8826    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
8827        let round = ReviewRound {
8828            round: 1,
8829            head: "deadbee".to_owned(),
8830            verified_head: None,
8831            verified_at: None,
8832            reviews: vec![ReviewRecord {
8833                attempts: 0,
8834                reviewer: 1,
8835                agent: "alpha".to_owned(),
8836                summary: String::new(),
8837                findings: Vec::new(),
8838                vote: None,
8839                failed: None,
8840                duration_ms: 0,
8841            }],
8842            e2e: Vec::new(),
8843            verify_retried: false,
8844            e2e_deferred: false,
8845            e2e_defer_reason: None,
8846            fix: None,
8847            blocking: 0,
8848            answered: 1,
8849            expected: 1,
8850            clean: true,
8851            progressed: false,
8852            vote_split: false,
8853            reconsideration: Vec::new(),
8854            verdict: None,
8855        };
8856        let state = state_with_round(round);
8857        let body = pr_message(&state, 'A').body;
8858        assert!(!body.contains("Open review findings"), "{body}");
8859        assert!(!body.contains("Declined"), "{body}");
8860    }
8861
8862    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
8863        let mut state = RunState::new(
8864            PathBuf::from("/repo"),
8865            "main".to_owned(),
8866            "abc1234".to_owned(),
8867            instruction.to_owned(),
8868            Config::default(),
8869        );
8870        state.candidates.push(Candidate {
8871            index: 0,
8872            label: 'A',
8873            agent: "alpha".to_owned(),
8874            branch: "magi/x/A".to_owned(),
8875            worktree: PathBuf::from("/wt"),
8876            summary: summary.to_owned(),
8877            stat: String::new(),
8878            files: 1,
8879            commits: 1,
8880            empty: false,
8881            failed: None,
8882            verified_noop: None,
8883            folded: false,
8884            duration_ms: 0,
8885        });
8886        state
8887    }
8888
8889    #[test]
8890    fn pr_message_describes_the_change_not_the_task() {
8891        let state = state_with_summary(
8892            "今回やってほしいこと: results projector を直す",
8893            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
8894        );
8895        let m = pr_message(&state, 'A');
8896        assert_eq!(m.title, "fix(web): batch the runs list reads");
8897        assert!(
8898            m.body.starts_with("## Summary\n\n- reads run.json once"),
8899            "{}",
8900            m.body
8901        );
8902        assert!(!m.body.contains("TITLE:"), "{}", m.body);
8903        let task_at = m.body.find("今回やってほしいこと").unwrap();
8904        let details_at = m.body.find("<details>").unwrap();
8905        assert!(
8906            details_at < task_at,
8907            "the task lives inside <details>: {}",
8908            m.body
8909        );
8910        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
8911        assert!(m.body.contains("magi:candidate-a"));
8912    }
8913
8914    #[test]
8915    fn pr_message_falls_back_to_the_task_without_a_title_line() {
8916        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
8917        let m = pr_message(&state, 'A');
8918        assert_eq!(m.title, "add retries");
8919        assert!(
8920            m.body.contains("## Summary\n\n- did some things"),
8921            "{}",
8922            m.body
8923        );
8924
8925        let none = RunState::new(
8926            PathBuf::from("/repo"),
8927            "main".to_owned(),
8928            "abc1234".to_owned(),
8929            "add retries".to_owned(),
8930            Config::default(),
8931        );
8932        let m = pr_message(&none, 'A');
8933        assert_eq!(m.title, "add retries");
8934        assert!(!m.body.contains("## Summary"), "{}", m.body);
8935    }
8936
8937    #[test]
8938    fn pr_message_refuses_the_candidate_commit_subject() {
8939        for bad in [
8940            "TITLE: magi: candidate A (uncommitted work)",
8941            "TITLE: chore: stuff (uncommitted work)",
8942            "TITLE:   ",
8943        ] {
8944            let state = state_with_summary("add retries", bad);
8945            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
8946        }
8947    }
8948
8949    #[test]
8950    fn pr_message_bounds_a_very_long_task_and_title() {
8951        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
8952        let state = state_with_summary(&long, "- nothing");
8953        let m = pr_message(&state, 'A');
8954        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
8955        assert!(!m.title.contains('\n'));
8956
8957        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
8958        let m = pr_message(&state, 'A');
8959        assert!(m.title.starts_with("feat: "));
8960        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
8961        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
8962    }
8963
8964    #[test]
8965    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
8966        // What magi itself writes stays English under any configured language,
8967        // so a future localisation of these headings fails here. (The agents'
8968        // own text is held to English by the prompt only; magi cannot check it.)
8969        let mut state = state_with_summary(
8970            "add retries",
8971            "TITLE: fix(web): batch reads\n- reads run.json once",
8972        );
8973        state.config.graph.language = "ja".to_owned();
8974        let m = pr_message(&state, 'A');
8975        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
8976
8977        // The task is the operator's own text: it goes in untouched, and the
8978        // fallback title (no summary) may be in its language too.
8979        let task = "今回やってほしいこと: results projector を直す";
8980        let mut state = state_with_summary(task, "- no title line");
8981        state.config.graph.language = "ja".to_owned();
8982        let m = pr_message(&state, 'A');
8983        assert_eq!(
8984            m.title,
8985            format!("chore: land candidate A of run {}", state.id)
8986        );
8987        assert!(
8988            m.body.contains(&format!(
8989                "<summary>Original task</summary>\n\n{task}\n\n</details>"
8990            )),
8991            "{}",
8992            m.body
8993        );
8994    }
8995
8996    #[test]
8997    fn pr_message_scrubs_home_paths_and_addresses() {
8998        let state = state_with_summary(
8999            "fix it in /Users/someone/src/x",
9000            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
9001        );
9002        let m = pr_message(&state, 'A');
9003        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
9004            assert!(!m.body.contains(leak), "{}", m.body);
9005        }
9006        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
9007    }
9008
9009    #[test]
9010    fn pr_message_survives_a_task_that_closes_details() {
9011        let state = state_with_summary("a </details> b", "TITLE: fix: x");
9012        let m = pr_message(&state, 'A');
9013        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
9014    }
9015
9016    #[test]
9017    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
9018        let cmd = manual_merge_command(
9019            MergeStyle::Squash,
9020            Path::new("/repo"),
9021            "b",
9022            "fix: \"quoted\" $(x) `y`\n\nbody",
9023        );
9024        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
9025    }
9026
9027    #[test]
9028    fn manual_merge_command_matches_the_configured_style() {
9029        let repo = Path::new("/repo");
9030        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
9031
9032        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
9033        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
9034
9035        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
9036        assert_eq!(
9037            squash,
9038            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
9039             \"Merge magi run 0832 (candidate A)\""
9040        );
9041
9042        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
9043        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
9044    }
9045
9046    #[test]
9047    fn a_nudge_gets_a_quarter_of_the_budget() {
9048        // The judge and implement budgets magi ships with.
9049        assert_eq!(retry_budget(secs(1200), true), secs(300));
9050        assert_eq!(retry_budget(secs(3600), true), secs(900));
9051    }
9052
9053    #[test]
9054    fn a_resent_prompt_keeps_the_whole_budget() {
9055        // The seat kept no context, so the retry is the original job again and
9056        // shortening it would only guarantee a second failure.
9057        assert_eq!(retry_budget(secs(1200), false), secs(1200));
9058        assert_eq!(retry_budget(secs(60), false), secs(60));
9059    }
9060
9061    #[test]
9062    fn the_floor_never_exceeds_the_original_budget() {
9063        // A short configured timeout must not be *raised* by the floor: the
9064        // operator asked for a bound, and a retry may not outlast the attempt
9065        // it is retrying.
9066        assert_eq!(retry_budget(secs(60), true), secs(60));
9067        assert_eq!(retry_budget(secs(480), true), secs(120));
9068        assert_eq!(retry_budget(secs(0), true), secs(0));
9069    }
9070
9071    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
9072        agent::CommandEvidence {
9073            id: "item1".to_owned(),
9074            description: "cargo test".to_owned(),
9075            exit_code,
9076            result_summary: String::new(),
9077            source: "codex".to_owned(),
9078        }
9079    }
9080
9081    #[test]
9082    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
9083        // No evidence is not the same fact as unconfirmed evidence: a
9084        // backend with no adapter, or a reply that ran no commands at all,
9085        // must not be misread as carrying a dangling job.
9086        assert!(!has_unconfirmed_command(&[]));
9087    }
9088
9089    #[test]
9090    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
9091        // Deliberately not a check on the exit code's *value*: a fixer
9092        // legitimately runs something that fails mid-iteration before it
9093        // succeeds, and that must never by itself reopen a valid report.
9094        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
9095        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
9096        assert!(!has_unconfirmed_command(&[
9097            evidence(Some(0)),
9098            evidence(Some(101))
9099        ]));
9100    }
9101
9102    #[test]
9103    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
9104        assert!(has_unconfirmed_command(&[
9105            evidence(Some(0)),
9106            evidence(None)
9107        ]));
9108    }
9109
9110    #[test]
9111    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
9112        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
9113        assert_eq!(
9114            verified_noop_claim(true, &[], text).as_deref(),
9115            Some("already fixed by b32cfc4, on main.")
9116        );
9117    }
9118
9119    #[test]
9120    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
9121        // A timeout or a bad exit code reads as the ordinary loss it is,
9122        // whatever the reply's own prose claims.
9123        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
9124        assert!(verified_noop_claim(false, &[], text).is_none());
9125    }
9126
9127    #[test]
9128    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
9129        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
9130        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
9131        // A confirmed command alongside the marker is fine.
9132        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
9133    }
9134
9135    #[test]
9136    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
9137        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
9138    }
9139
9140    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
9141    /// pair, in order, labelled A, B, C, ...
9142    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
9143        runner.state.candidates = shape
9144            .iter()
9145            .enumerate()
9146            .map(|(i, &(empty, verified))| Candidate {
9147                index: i,
9148                label: (b'A' + i as u8) as char,
9149                agent: "sonnet".to_owned(),
9150                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
9151                worktree: PathBuf::from(format!("/wt/{i}")),
9152                summary: String::new(),
9153                stat: String::new(),
9154                files: 0,
9155                commits: 0,
9156                empty,
9157                failed: None,
9158                verified_noop: verified.map(str::to_owned),
9159                duration_ms: 0,
9160                folded: false,
9161            })
9162            .collect();
9163    }
9164
9165    #[test]
9166    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
9167        ask_test_home();
9168        let mut runner = runner_at(RunStatus::Implementing);
9169        set_candidates(
9170            &mut runner,
9171            &[
9172                (true, Some("already on main at b32cfc4")),
9173                (true, Some("same fix, see the existing test")),
9174            ],
9175        );
9176
9177        runner
9178            .after_implement()
9179            .expect("a verified no-op is not an error");
9180
9181        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
9182    }
9183
9184    #[test]
9185    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
9186        ask_test_home();
9187        let mut runner = runner_at(RunStatus::Implementing);
9188        // Candidate A declares a verified no-op; candidate B simply wrote
9189        // nothing and said nothing about why. One candidate's claim is not
9190        // the whole run's agreement.
9191        set_candidates(
9192            &mut runner,
9193            &[(true, Some("already on main at b32cfc4")), (true, None)],
9194        );
9195
9196        let err = runner
9197            .after_implement()
9198            .expect_err("an unverified empty candidate must still fail the run");
9199
9200        assert!(
9201            err.to_string().contains("no candidate produced a change"),
9202            "{err}"
9203        );
9204        assert_eq!(runner.state.status, RunStatus::Failed);
9205    }
9206
9207    #[test]
9208    fn after_implement_still_fails_an_ordinary_all_empty_run() {
9209        ask_test_home();
9210        let mut runner = runner_at(RunStatus::Implementing);
9211        set_candidates(&mut runner, &[(true, None), (true, None)]);
9212
9213        let err = runner
9214            .after_implement()
9215            .expect_err("no candidate declared anything; this is an ordinary failure");
9216
9217        assert!(
9218            err.to_string().contains("no candidate produced a change"),
9219            "{err}"
9220        );
9221        assert_eq!(runner.state.status, RunStatus::Failed);
9222    }
9223}