Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54    Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58    ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62/// How much verification output is kept and fed back to the fixer.
63const OUTPUT_TAIL: usize = 8_000;
64
65/// Bytes of a failing command's output kept in an event, so the reason a run
66/// stopped is readable from the report without opening `run.json`.
67const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
70/// command's pid before releasing the build cache's lease.
71const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
74/// command's pid to actually exit before giving up and releasing anyway.
75///
76/// A timeout means the process was asked to die (`kill_on_drop`,
77/// `start_kill`), not that it already has — on Windows in particular that can
78/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
79/// the instant the command returns would let the very next acquirer (this
80/// run's own next round, another run's verification, the janitor's prune)
81/// start touching the same directory while it might still be writing to it,
82/// so this polls the actual pid — real confirmation, not a fixed guess —
83/// until it is gone or this ceiling is reached. It is still not full
84/// process-tree reaping: a grandchild the timed-out process spawned and that
85/// outlives it independently is invisible to a pid check, and continuing to
86/// observe and collect *that* stays a different piece of work with its own
87/// owner. Set generously because the common case returns early the moment
88/// the pid is confirmed gone, not because every timeout pays this in full.
89const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91/// Consecutive review rounds with no tree progress (see
92/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
93/// instead of spending the rest of the round budget.
94///
95/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
96/// legitimately finds nothing left to change (its previous round's fix already
97/// covered it, and this round's reviewers re-raised only nits) looks the same
98/// as one that is spinning, for exactly one round. Two in a row is where the
99/// two stop being distinguishable, and a review round on this workload has
100/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
101/// third attempt at a tree that has not moved twice running is pure cost.
102/// This does not touch `review_rounds` itself, which stays the operator's
103/// call.
104pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
107/// a base that moved before giving up and leaving the run `Blocked` for a
108/// person.
109///
110/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
111/// that keeps moving faster than a run can catch it is not something more
112/// rebasing fixes, it is a person's call. Not the same *number as*
113/// `land_rounds` - this budget is spent before a pull request exists, land's
114/// after - but bounded for the identical reason, so it uses the same
115/// default. Counted across both call sites in [`Runner::finish_after_tally`]
116/// (once before review, once before the gate), because either one finding
117/// the base still moving is the same signal.
118const BASE_SYNC_ROUNDS: usize = 4;
119
120/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
121/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
122/// reply held no [`FixReport`].
123///
124/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
125/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
126/// "I'll pause here until the `cargo make check` background run reports
127/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
128/// No `FixReport` was ever collected from that seat, and the run moved on to
129/// the next review round regardless.
130///
131/// Bounded independently of `review_rounds` and `graph.retries`: this
132/// recovers one seat's missing report mid-round, not a new round of review or
133/// an ordinary parse retry, and must not itself become the unbounded wait the
134/// rest of this module exists to avoid.
135const MAX_FIX_CONTINUATIONS: usize = 2;
136
137/// One queued agent invocation.
138///
139/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
140/// CLI hung up on its own stream is asked again from the same job rather than
141/// rebuilt from scratch. See [`Runner::resume_undelivered`].
142#[derive(Clone)]
143struct SeatJob {
144    spec: AgentSpec,
145    seat: SeatState,
146    cwd: PathBuf,
147    prompt: String,
148    timeout: Duration,
149    allow_write: bool,
150    sessions: bool,
151    artifacts: PathBuf,
152    stem: String,
153    /// The prompt for a seat that has been handed to another roster agent
154    /// (a fresh session): everything the original seat would have
155    /// remembered. `None` when `prompt` already carries it, as the first
156    /// ranking and the implement prompt do. Never a resume-style prompt.
157    handover: Option<String>,
158}
159
160/// How the graph reads one agent invocation.
161///
162/// Quota is split out from an ordinary failure on purpose: a rate-limited call
163/// is known to fail again if retried now, so the retry loop must not spend an
164/// attempt on it. `Dropped` is split out for the opposite reason: unlike
165/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
166/// error JSON, never the agent's answer — a caller that matched only
167/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
168/// left to read that JSON as if it were usable output. `resume_undelivered`
169/// is the only caller that acts on it; everywhere else it is reported like an
170/// ordinary failure.
171enum AgentOutcome {
172    /// A usable output.
173    Ok(AgentOutput),
174    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
175    Quota(AgentOutput),
176    /// The CLI hung up on its own stream after billed work. See
177    /// [`agent::AgentOutput::work_undelivered`].
178    Dropped(AgentOutput),
179    /// Any other failure: a timeout, a bad exit code, an empty reply.
180    Failed(String),
181    /// The agent was never spawned: a shared resource it needs (the build
182    /// cache's lease) stayed busy past `[graph] timeout_resource_wait`, or the
183    /// wait was abandoned because the run is parking. Not a failure of the
184    /// agent: it must not advance a fallback chain, enter seat history, cost
185    /// quota or spend an attempt. [`RunState::resource_block`] says why.
186    ResourceBlocked(String),
187}
188
189/// A request to park the run at its next node boundary.
190///
191/// Cloning is how the request travels: the loop keeps one handle and hands a
192/// clone to each [`Runner`], and every clone points at the same flag. There
193/// is no channel because there is nothing to send - the only message is
194/// "park", it is idempotent, and a flag cannot be missed by a receiver that
195/// was not listening yet.
196///
197/// The boundary is what makes this cheap. Every node writes the run's state
198/// before the next one starts, and every node skips what is already recorded:
199/// `prep` returns early once candidates exist, `implement` asks only the seats
200/// with nothing on disk, `judge` returns early once judgements exist. So a
201/// parked run resumes into exactly the node it stopped before, and no agent
202/// work is thrown away. Killing the process mid-node, by contrast, loses
203/// whatever the seats in flight had not yet written - which for an implement
204/// wave is an hour of paid work.
205///
206/// A [`Runner`] watches two independent handles of this type - see
207/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
208/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
209/// clone covering the whole daemon's lifetime and is never asked to un-park,
210/// which is correct exactly because nothing is dispatched after it fires.
211/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
212/// that parks for an interrupted task must go on to run other tasks
213/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
214/// reusing the daemon-wide one.
215#[derive(Debug, Clone, Default)]
216pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
217
218impl Pause {
219    /// A pause nobody has asked for yet.
220    #[must_use]
221    pub fn new() -> Self {
222        Self::default()
223    }
224
225    /// Ask the run to park at its next node boundary. Idempotent.
226    pub fn park(&self) {
227        self.0.store(true, Ordering::SeqCst);
228    }
229
230    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
231    /// fold into the run's own `park` event - so an operator reading the run
232    /// later knows this was a deliberate interrupt rather than a shutdown or
233    /// a binary swap. The first reason recorded wins; a park already in
234    /// flight is not relabelled by a second, unrelated request.
235    pub fn park_because(&self, reason: impl Into<String>) {
236        self.park_for(ParkCause::Interrupt {
237            reason: reason.into(),
238        });
239    }
240
241    /// [`Pause::park`], recording the structured cause. The first cause
242    /// recorded wins, like [`Pause::park_because`].
243    pub fn park_for(&self, cause: ParkCause) {
244        let mut reason_guard = self
245            .1
246            .lock()
247            .unwrap_or_else(std::sync::PoisonError::into_inner);
248        if reason_guard.is_none() {
249            *reason_guard = Some(cause);
250        }
251        drop(reason_guard);
252        self.park();
253    }
254
255    /// Has a park been asked for?
256    #[must_use]
257    pub fn parked(&self) -> bool {
258        self.0.load(Ordering::SeqCst)
259    }
260
261    /// Why the park was asked for, when the caller used [`Pause::park_because`].
262    #[must_use]
263    pub fn reason(&self) -> Option<String> {
264        self.cause().map(|c| c.label())
265    }
266
267    /// The structured cause, when one was recorded.
268    #[must_use]
269    pub fn cause(&self) -> Option<ParkCause> {
270        self.1
271            .lock()
272            .unwrap_or_else(std::sync::PoisonError::into_inner)
273            .clone()
274    }
275}
276
277/// Drives one run.
278pub struct Runner {
279    /// Run state; public so the CLI can report on it.
280    pub state: RunState,
281    roles: ResolvedRoles,
282    sem: Arc<Semaphore>,
283    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
284    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
285    /// this is never the same handle as `interrupt`.
286    pause: Pause,
287    /// Set when `magi serve`'s interrupt scheduler wants this specific run
288    /// parked at its next node boundary, to let a task marked
289    /// [`crate::queue::Task::interrupt`] run alone before this one carries
290    /// on. Unlike `pause`, a fresh, unshared handle per run - see
291    /// [`Runner::watch_interrupt`].
292    interrupt: Pause,
293}
294
295/// Where the branch's own commits start: its merge base with the base branch
296/// as the remote has it now, else with the recorded `base_commit`. A branch
297/// rebased onto a base that moved past `base_commit` would otherwise count
298/// the base's commits as its own under `base_commit..branch`.
299async fn review_base(
300    repo: &Path,
301    remote: &str,
302    base_branch: &str,
303    base_commit: &str,
304    branch: &str,
305) -> String {
306    review_base_checked(repo, remote, base_branch, base_commit, branch)
307        .await
308        .0
309}
310
311/// [`review_base`] plus whether the base was read from a freshly fetched
312/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
313/// never older than `base_commit`'s view of the base), but the answer is then
314/// not trusted to rewrite a pull request's title.
315async fn review_base_checked(
316    repo: &Path,
317    remote: &str,
318    base_branch: &str,
319    base_commit: &str,
320    branch: &str,
321) -> (String, bool) {
322    let tracking = format!("{remote}/{base_branch}");
323    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
324    if git::rev_exists(repo, &tracking).await
325        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
326        && !mb.is_empty()
327    {
328        return (mb, fresh);
329    }
330    let mb = git::merge_base(repo, base_commit, branch)
331        .await
332        .ok()
333        .filter(|mb| !mb.is_empty())
334        .unwrap_or_else(|| base_commit.to_owned());
335    (mb, false)
336}
337
338/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
339/// when git cannot say or finds nothing: a stale list is better than a wrong
340/// or empty one.
341pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
342    if !is_review_run(state) {
343        return;
344    }
345    let base = review_base(
346        &state.repo,
347        &state.config.merge.remote,
348        &state.base_branch,
349        &state.base_commit,
350        branch,
351    )
352    .await;
353    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
354        && !subjects.is_empty()
355        && state.reviewed_commits.as_ref() != Some(&subjects)
356    {
357        state.reviewed_commits = Some(subjects);
358        state.save().ok();
359    }
360}
361
362/// Subjects of the base's commits between the recorded start and the branch's
363/// merge base: what a stale `base_commit..branch` would have mistaken for the
364/// branch's own work.
365async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
366    let (base, trusted) = review_base_checked(
367        &state.repo,
368        &state.config.merge.remote,
369        &state.base_branch,
370        &state.base_commit,
371        branch,
372    )
373    .await;
374    if !trusted {
375        return None;
376    }
377    git::subjects(&state.repo, &state.base_commit, &base)
378        .await
379        .ok()
380}
381
382/// May an adopted pull request's title be replaced with `computed`? Only when
383/// it is empty, magi's own shape, or a base commit's subject that leaked in;
384/// a title a person wrote stays. Never when `computed` is itself a leak.
385pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
386    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
387    if is_leak(computed) {
388        return false;
389    }
390    let cur = current.trim();
391    cur.is_empty()
392        || cur.starts_with(REVIEW_PROMPT_OPENING)
393        || cur.starts_with("chore: land candidate")
394        || cur.starts_with("magi: candidate")
395        || is_leak(cur)
396}
397
398/// The commit a run branches from: the base branch as the remote has it.
399///
400/// Two failures this replaces. A run used to branch off `HEAD` and so refused
401/// to start on a dirty tree, which made `magi serve` decline every task for as
402/// long as the operator had work in progress - most of the time. Branching off
403/// the *local* base branch fixed that and introduced a worse one: `land` merges
404/// the winner on GitHub, nothing updates the local ref, and the next run
405/// branches off a base missing everything the previous runs landed. Two tasks
406/// in a row from a phone would have had the second silently re-implementing
407/// against stale code and opening a pull request that reverted the first.
408///
409/// Only refs move here - no checkout, no local branch, no merge - so it is safe
410/// with uncommitted work in the tree. A machine with no network still starts:
411/// the fetch may fail and the local tip is used with a warning, because
412/// refusing to run offline is a worse failure than running against a base the
413/// operator can see for themselves.
414///
415/// One function, called by both entry points. Two answers to "where does a run
416/// branch from" is the kind of drift nobody notices until a diff is wrong.
417/// Bring the local `branch` in line with `<remote>/<branch>` before a review
418/// checks it out.
419///
420/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
421/// anything other than plain `git push` from this checkout (a jj colocated
422/// workspace, another clone) moves only the remote-tracking ref - so the local
423/// one can be a stale placeholder. It moves only when local is behind the remote or is an
424/// empty placeholder that diverged from it; unpushed local work is kept, and a real
425/// divergence is refused rather than guessed at.
426async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
427    let tracking = format!("{remote}/{branch}");
428    let fetched = git::fetch(repo, remote, branch).await;
429    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
430    let local_exists = git::branch_exists(repo, branch).await?;
431    if !fresh {
432        if !local_exists {
433            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
434        }
435        tracing::warn!(
436            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
437        );
438        return Ok(());
439    }
440    let remote_sha = git::rev_parse(repo, &tracking).await?;
441    if !local_exists {
442        git::git(repo, &["branch", branch, &tracking]).await?;
443        return Ok(());
444    }
445    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
446    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
447        return Ok(());
448    }
449    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
450        // Diverged. `reconcile` settles it only when it can prove nothing is
451        // lost: a local tip that is the remote's change rebased is pushed over
452        // it (lease pinned to the tip read here), a tip whose every extra
453        // commit is empty is a placeholder the remote's work replaced, and
454        // anything else is two different changes - a question for a person.
455        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
456            .await?
457        {
458            crate::reconcile::Reconciliation::Pushed => {
459                tracing::warn!(
460                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
461                    short(&local_sha),
462                    short(&remote_sha)
463                );
464                return Ok(());
465            }
466            crate::reconcile::Reconciliation::Placeholder => {}
467            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
468        }
469    }
470    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
471    if !out.ok() {
472        bail!(
473            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
474            short(&local_sha),
475            short(&remote_sha),
476            out.stderr
477        );
478    }
479    tracing::warn!(
480        "local `{branch}` was stale: fast-forwarded {} -> {}",
481        short(&local_sha),
482        short(&remote_sha)
483    );
484    Ok(())
485}
486
487async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
488    let tracking = format!("{remote}/{base_branch}");
489    let fetched = git::fetch(repo, remote, base_branch).await;
490    if let Ok(out) = &fetched
491        && out.ok()
492        && git::rev_exists(repo, &tracking).await
493    {
494        return git::rev_parse(repo, &tracking).await;
495    }
496    let why = match &fetched {
497        Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
498        Ok(_) => format!("{remote} has no {base_branch}"),
499        Err(e) => e.to_string(),
500    };
501    // No fallback to the local branch: it may be behind, and branching off an
502    // old commit is the stale-checkout bug this check exists to prevent.
503    bail!(
504        "cannot read {tracking} ({why}); refusing to branch off the local \
505         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
506         base / remote in magi.toml"
507    )
508}
509
510/// Exclusive claim on one run's `magi fix` step, released on drop — including
511/// on an early return or a panic.
512///
513/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
514/// manual `magi fix` invocations against the same run are otherwise
515/// invisible to each other and would race to remove and recreate the same
516/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
517/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
518/// which is only ever reclaimed later, out of band, by
519/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
520/// `magi fix` invocation is not necessarily running under either of those, so
521/// nothing would ever sweep a lock a killed or crashed process left behind.
522/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
523/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
524/// lease use: an unreadable or unparsable pid, or a liveness query the
525/// platform cannot answer, reads as alive and the lock is left in place.
526struct FixClaim {
527    path: PathBuf,
528}
529
530impl FixClaim {
531    fn acquire(dir: &Path) -> Result<Self> {
532        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
533        let path = dir.join("fix.lock");
534        match Self::create(&path) {
535            Ok(claim) => Ok(claim),
536            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
537                if Self::reclaim_if_dead(&path) {
538                    Self::create(&path).with_context(|| format!("lock {}", path.display()))
539                } else {
540                    bail!(
541                        "another `magi fix` is already running for this run ({} exists)",
542                        path.display()
543                    )
544                }
545            }
546            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
547        }
548    }
549
550    fn create(path: &Path) -> std::io::Result<Self> {
551        let mut f = std::fs::OpenOptions::new()
552            .write(true)
553            .create_new(true)
554            .open(path)?;
555        use std::io::Write as _;
556        // Read back by `reclaim_if_dead` on a later, stuck invocation.
557        writeln!(f, "{}", std::process::id())?;
558        Ok(Self {
559            path: path.to_owned(),
560        })
561    }
562
563    /// True if the lock named a process confirmed dead, in which case it was
564    /// also removed. Never true on an unreadable file, an unparsable pid, or
565    /// a liveness query the platform cannot answer — see this type's own doc.
566    fn reclaim_if_dead(path: &Path) -> bool {
567        let dead = std::fs::read_to_string(path)
568            .ok()
569            .and_then(|body| body.trim().parse::<u32>().ok())
570            .is_some_and(|pid| !crate::proc::pid_alive(pid));
571        dead && std::fs::remove_file(path).is_ok()
572    }
573}
574
575impl Drop for FixClaim {
576    fn drop(&mut self) {
577        let _ = std::fs::remove_file(&self.path);
578    }
579}
580
581impl Runner {
582    /// Start a fresh run against `repo`.
583    pub async fn start(
584        repo: &Path,
585        instruction: String,
586        config: Config,
587        origin: Origin,
588    ) -> Result<Self> {
589        Self::start_naming(repo, instruction, "", config, origin).await
590    }
591
592    /// [`Runner::start`] for a queued task: `also_scan` (the task's title) is
593    /// searched for branch and commit references along with the instruction,
594    /// since a task may name the work it is about only in its title.
595    pub async fn start_naming(
596        repo: &Path,
597        instruction: String,
598        also_scan: &str,
599        config: Config,
600        origin: Origin,
601    ) -> Result<Self> {
602        let repo = git::toplevel(repo).await?;
603        let missing = agent::missing_programs(&config.agents);
604        if !missing.is_empty() {
605            bail!(
606                "these agent programs are not on PATH: {}. Fix the roster in \
607                 magi.toml or install them.",
608                missing.join(", ")
609            );
610        }
611        let base_branch =
612            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
613                .await?;
614        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
615        let roles = config.resolve_roles()?;
616        let max_parallel = config.graph.max_parallel.max(1);
617        // A task that points at work already in the repository starts from
618        // it; what the repository says about each reference is recorded.
619        let seeds = refs::resolve(
620            &repo,
621            &base_commit,
622            &config.merge.remote,
623            &format!("{also_scan}\n{instruction}"),
624        )
625        .await;
626        refs::plan(&repo, &seeds).await?;
627        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
628        // Recorded before the first save, so a crash right after minting
629        // cannot leave a run with no origin. Legibility only: nothing reads it
630        // to decide anything.
631        state.origin = Some(origin);
632        for seed in &seeds {
633            state.event(
634                "seed",
635                refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
636            );
637        }
638        state.seeds = seeds;
639        state.event("start", format!("run {} created", state.id));
640        state.save()?;
641        Ok(Self {
642            state,
643            roles,
644            sem: Arc::new(Semaphore::new(max_parallel)),
645            pause: Pause::new(),
646            interrupt: Pause::new(),
647        })
648    }
649
650    /// Open a review-only run against work that already exists on `branch`.
651    ///
652    /// The expensive half of the graph is the implement wave — measured at
653    /// 111 and 134 internal tool-loop turns on this repository, against a
654    /// handful for a judge or a reviewer. The cheap half is worth running on
655    /// hand-written work too, and there was no way to reach it.
656    ///
657    /// No new state and no schema change are needed: a run with **one** viable
658    /// candidate and a tally already decided degrades `execute` to exactly
659    /// review → gate → merge, because `judge` skips a single-candidate field,
660    /// `deliberate` has fewer than two first choices to reconcile, `vote`
661    /// returns early, `tally` is already present and `fold_losers` has no
662    /// losers. Resuming such a run therefore does the right thing as well.
663    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
664        Self::review_taking_over(repo, branch, config, None, origin).await
665    }
666
667    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
668    /// at the same task still has `branch` checked out, its worktree is
669    /// released first if that is safe (see [`crate::handover`]), and the
670    /// review refuses with the reason if it is not. `None` is a hand-run
671    /// review: it has no earlier attempts, so only a worktree of a dead run
672    /// magi recorded itself can be released.
673    pub async fn review_taking_over(
674        repo: &Path,
675        branch: &str,
676        config: Config,
677        takeover: Option<crate::handover::Takeover>,
678        origin: Origin,
679    ) -> Result<Self> {
680        let repo = git::toplevel(repo).await?;
681        let missing = agent::missing_programs(&config.agents);
682        if !missing.is_empty() {
683            bail!(
684                "these agent programs are not on PATH: {}. Fix the roster in \
685                 magi.toml or install them.",
686                missing.join(", ")
687            );
688        }
689        let base_branch =
690            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
691                .await?;
692        if base_branch == branch {
693            bail!("`{branch}` is the base branch; there is nothing to review against");
694        }
695        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
696
697        let roles = config.resolve_roles()?;
698        let max_parallel = config.graph.max_parallel.max(1);
699        let mut state = RunState::new(
700            repo.clone(),
701            base_branch,
702            base_commit.clone(),
703            String::new(),
704            config,
705        );
706        state.origin = Some(origin);
707
708        // Released before anything else touches the branch: a stale local
709        // branch is moved with `git branch -f`, which git refuses while an
710        // earlier attempt's worktree still has it checked out. Everything
711        // after this point that can fail puts the old run back.
712        // A hand-run review has no task, hence no earlier attempts, but a
713        // worktree of a dead run magi made may still be released.
714        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
715            earlier: Vec::new(),
716            home: crate::run::home(),
717            choice: None,
718            task_instruction: None,
719        });
720        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
721        if let Some(released) = &released {
722            state.event(
723                "release",
724                format!(
725                    "took `{branch}` over from run {}: its worktree was released: {}",
726                    crate::run::short_of(&released.old_id),
727                    released.audit
728                ),
729            );
730        }
731        // The owner's answer to an earlier divergence question is applied
732        // here: after the release (git will not move a checked-out branch)
733        // and before the sync that would otherwise ask again.
734        if let Some(choice) = takeover.choice.as_ref()
735            && let Err(e) =
736                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
737                    .await
738        {
739            if let Some(released) = &released {
740                released.restore(&repo, branch).await;
741            }
742            return Err(e.context("applying the owner's answer about the diverged branch"));
743        }
744        // An earlier attempt of this task that published the task head, and
745        // whose work is the branch under review, hands that head over: the
746        // review run updates the same pull request. Without one (a legacy
747        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
748        // own head and its own open pull request is adopted as before.
749        for id in takeover.earlier.iter().rev() {
750            if let Ok(old) = RunState::load_under(id, &takeover.home)
751                && let Some(head) = old.pr_head.clone()
752                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
753            {
754                state.pr_head = Some(head);
755                break;
756            }
757        }
758        let opened = Self::open_review(
759            &repo,
760            branch,
761            state,
762            roles,
763            max_parallel,
764            base_commit,
765            &takeover,
766        )
767        .await;
768        if opened.is_err()
769            && let Some(released) = &released
770        {
771            released.restore(&repo, branch).await;
772        }
773        opened
774    }
775
776    /// What the newest earlier attempt that reviewed `branch` leaves for this
777    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
778    /// no earlier attempts. Reads the earlier record only: ownership of its
779    /// worktree and branch is [`crate::handover`]'s business.
780    async fn inherit(
781        repo: &Path,
782        branch: &str,
783        base_commit: &str,
784        takeover: &crate::handover::Takeover,
785    ) -> Option<crate::recovery::RecoveryRecord> {
786        use crate::recovery::{Relation, assess};
787        let newest = takeover.earlier.last()?;
788        let mut chosen = None;
789        for id in takeover.earlier.iter().rev() {
790            if let Ok(old) = RunState::load_under(id, &takeover.home)
791                && !old.reviews.is_empty()
792                && (old.winner().is_some_and(|w| w.branch == branch)
793                    || old.pr_head.as_deref() == Some(branch))
794            {
795                chosen = Some(old);
796                break;
797            }
798        }
799        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
800            .await
801            .unwrap_or_default();
802        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
803            Some(last) if last.head == tip => Relation::Same,
804            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
805            _ => Relation::Diverged,
806        };
807        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
808        Some(assess(from, chosen.as_ref(), base_commit, relation))
809    }
810
811    /// The half of [`Runner::review_taking_over`] that can fail after an
812    /// earlier attempt's worktree was released.
813    async fn open_review(
814        repo: &Path,
815        branch: &str,
816        mut state: RunState,
817        roles: ResolvedRoles,
818        max_parallel: usize,
819        base_commit: String,
820        takeover: &crate::handover::Takeover,
821    ) -> Result<Self> {
822        sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
823        // The commit subjects are the closest thing to a task statement that
824        // existing work carries, and the reviewers are told as much.
825        let start = review_base(
826            repo,
827            &state.config.merge.remote,
828            &state.base_branch,
829            &base_commit,
830            branch,
831        )
832        .await;
833        let log = git::log_oneline(repo, &start, branch)
834            .await
835            .unwrap_or_default();
836        let log_text = if log.trim().is_empty() {
837            "(no commit messages)"
838        } else {
839            log.trim()
840        };
841        let mut instruction = format!(
842            "Review the work already on branch `{branch}`. There is no task \
843             statement: what the change claims to do is whatever its commits \
844             say.\n\n{log_text}"
845        );
846        // A retry of a task whose earlier attempt left reviews behind carries
847        // that attempt's evidence, and the task's own statement with it: the
848        // completion criteria are the task's, not whatever the commits claim.
849        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
850            if let Some(task) = takeover
851                .task_instruction
852                .as_deref()
853                .filter(|t| !t.trim().is_empty())
854                && rec.reviewed_head.is_some()
855            {
856                instruction = format!(
857                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
858                    task.trim()
859                );
860                rec.criteria_inherited = true;
861            }
862            state.event("recovery", rec.summary());
863            state.recovery = Some(rec);
864        }
865        state.instruction = instruction;
866        state.reviewed_commits = Some(
867            git::subjects(repo, &start, branch)
868                .await
869                .unwrap_or_default(),
870        );
871
872        // An attached worktree, so the fixer's commits land on the branch under
873        // review rather than on a detached head nobody will look at again.
874        let worktree = state.worktree_root().join("under-review");
875        if let Some(parent) = worktree.parent() {
876            tokio::fs::create_dir_all(parent).await.ok();
877        }
878        let path = worktree.to_string_lossy().to_string();
879        git::git(repo, &["worktree", "add", &path, branch])
880            .await
881            .with_context(|| {
882                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
883            })?;
884
885        let setup = match ensure_setup_config(&mut state, repo).await {
886            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
887            Err(e) => Err(e),
888        };
889        if let Err(e) = setup {
890            git::worktree_remove(repo, &worktree).await.ok();
891            return Err(e);
892        }
893        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
894            .await
895            .unwrap_or(0);
896        if commits == 0 {
897            git::worktree_remove(repo, &worktree).await.ok();
898            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
899        }
900        let files = git::changed_files(&worktree, &base_commit, "HEAD")
901            .await
902            .map(|f| f.len())
903            .unwrap_or(0);
904        if files == 0
905            && let (Ok(head_tree), Ok(base_tree)) = (
906                git::tree_of(&worktree, "HEAD").await,
907                git::tree_of(&worktree, &base_commit).await,
908            )
909            && head_tree == base_tree
910        {
911            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
912            git::worktree_remove(repo, &worktree).await.ok();
913            bail!(
914                "`{branch}` at {} has a tree identical to base {}; this usually means \
915                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
916                 against `{}/{branch}`) rather than an empty change",
917                short(&head),
918                short(&base_commit),
919                state.config.merge.remote
920            );
921        }
922        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
923            .await
924            .unwrap_or_default();
925
926        state.candidates.push(Candidate {
927            index: 0,
928            label: 'A',
929            // Not an agent id on purpose: nothing in the roster wrote this, and
930            // the stats tables must not credit anyone with a win for it.
931            agent: EXISTING_BRANCH.to_owned(),
932            branch: branch.to_owned(),
933            worktree,
934            summary: String::new(),
935            stat,
936            files,
937            commits,
938            empty: false,
939            failed: None,
940            verified_noop: None,
941            duration_ms: 0,
942            folded: false,
943        });
944        state.tally = Some(Tally {
945            first_choice: BTreeMap::from([('A', 0)]),
946            borda: BTreeMap::new(),
947            winner: 'A',
948            rankings: 0,
949            unanimous_initial: false,
950            deliberated: false,
951            changed_votes: 0,
952            unanimous_final: false,
953            tie_break: None,
954            // No panel sat, so no quorum applies. Zero judges is the correct
955            // number for work that never competed, and must not be reported as
956            // a collapsed panel.
957            judges: 0,
958            present: 0,
959            quorum: 0,
960            met_quorum: true,
961            uncontested: Some("review-only run: nothing competed".to_owned()),
962        });
963        state.status = RunStatus::Reviewing;
964        state.event(
965            "start",
966            format!(
967                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
968                state.id
969            ),
970        );
971        state.save()?;
972        Ok(Self {
973            state,
974            roles,
975            sem: Arc::new(Semaphore::new(max_parallel)),
976            pause: Pause::new(),
977            interrupt: Pause::new(),
978        })
979    }
980
981    /// Reopen an existing run.
982    pub fn resume(id: &str) -> Result<Self> {
983        let state = RunState::load(id)?;
984        if let Some(to) = &state.released_to {
985            bail!(
986                "run {} cannot be resumed: its worktree was released to run {}",
987                state.short(),
988                crate::run::short_of(to)
989            );
990        }
991        let roles = state.config.resolve_roles()?;
992        let max_parallel = state.config.graph.max_parallel.max(1);
993        Ok(Self {
994            state,
995            roles,
996            sem: Arc::new(Semaphore::new(max_parallel)),
997            pause: Pause::new(),
998            interrupt: Pause::new(),
999        })
1000    }
1001
1002    /// Walk the graph to a terminal state, skipping nodes already recorded.
1003    ///
1004    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1005    /// the phone - ends here, so this is the one place a run that ended
1006    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1007    /// notification centre. Best-effort: see [`crate::notices::raise`].
1008    pub async fn execute(&mut self) -> Result<()> {
1009        let result = self.execute_graph().await;
1010        self.mark_driver_exited();
1011        let ended = if result.is_err() {
1012            Some(crate::notices::run_stopped(&self.state.id, &self.state))
1013        } else {
1014            crate::notices::run_ended(&self.state)
1015        };
1016        if let Some(notice) = ended {
1017            crate::notices::raise(notice);
1018        }
1019        result
1020    }
1021
1022    /// Record that this process no longer drives the run, so its pid (a
1023    /// daemon's outlives the run) is not read as a live driver.
1024    ///
1025    /// Written onto the record as it is on disk, never this copy: another
1026    /// process may have resumed the run (recording its own pid and clearing
1027    /// the flag) or released its worktree since this copy was read, and
1028    /// saving over that would mark a running driver dead. Only a record still
1029    /// naming this process as the driver is touched.
1030    fn mark_driver_exited(&mut self) {
1031        self.state.driver_exited = true;
1032        let pid = std::process::id();
1033        let Ok(_lock) = crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) else {
1034            return;
1035        };
1036        let Ok(mut disk) = RunState::load(&self.state.id) else {
1037            return;
1038        };
1039        if disk.released_to.is_some() || disk.driver_pid != Some(pid) || disk.driver_exited {
1040            return;
1041        }
1042        disk.driver_exited = true;
1043        if let Err(e) = disk.save() {
1044            tracing::warn!("could not record that run {} stopped: {e:#}", self.state.id);
1045        }
1046    }
1047
1048    /// Validate ownership and persist the driver claim in one critical section.
1049    fn claim_driver(&mut self) -> Result<()> {
1050        let _lock = crate::pr_head::owner_lock(&crate::run::home(), &self.state.id)?;
1051        let disk = RunState::load(&self.state.id)?;
1052        crate::pr_head::ensure_owner(&disk)?;
1053        // Moving again, so it is no longer parked. Set before the walk rather
1054        // than in `resume`, so every way of re-entering the graph clears it
1055        // and a card cannot claim a run is waiting to be resumed while the
1056        // agents are already working.
1057        self.state.parked = false;
1058        self.state.resource_wait = None;
1059        self.state.resource_block = None;
1060        // Any seat this state still lists as answering belongs to whatever
1061        // process last drove this run — this one included, if it crashed
1062        // mid-wave. Cleared and flushed immediately, before anything else
1063        // runs, so a resume can never show a seat as live when nothing is
1064        // asking it anything yet; the node that actually dispatches the next
1065        // wave repopulates it.
1066        self.state.clear_active();
1067        // Recorded in the same spot, and flushed together with the clear
1068        // above: this is the pid a reader checks (`RunState::liveness`) when
1069        // no daemon claim exists to answer "is a process still driving this
1070        // run" — a plain `magi run` / `magi review` typed into a terminal
1071        // claims nothing there. Always overwritten, never only-if-absent, so
1072        // a resumed run's stale pid from a previous, possibly-dead process
1073        // can never survive into this one's own report. Unlike
1074        // `clear_active`, this changes on every single `execute()` call, so
1075        // the save below is now unconditional rather than only-if-cleared.
1076        //
1077        // `driver_started_at` is recorded in the same breath, from this same
1078        // pid, so `liveness` can tell a live pid that is genuinely still us
1079        // apart from one the OS has since handed to an unrelated process —
1080        // see that field's own doc for why the pid alone is not enough.
1081        // A resume that raced a takeover: the record on disk says the worktree
1082        // was handed to a later run after this copy was read. Saving over it
1083        // would erase that and drive a run with nothing to run in.
1084        if let Some(to) = &disk.released_to {
1085            bail!(
1086                "run {} cannot continue: its worktree was released to run {}",
1087                self.state.short(),
1088                crate::run::short_of(to)
1089            );
1090        }
1091        let pid = std::process::id();
1092        self.state.driver_pid = Some(pid);
1093        self.state.driver_started_at = crate::proc::process_started_at(pid);
1094        self.state.driver_exited = false;
1095        self.state.begin_pass();
1096        self.state.save()?;
1097        Ok(())
1098    }
1099
1100    async fn execute_graph(&mut self) -> Result<()> {
1101        self.claim_driver()?;
1102        // Publication already passed review and gate. Resume only that node:
1103        // syncing to a newer base here could change the verified commit.
1104        if self.state.pr_publication_pending {
1105            self.merge().await?;
1106            self.state.save()?;
1107            self.settle_questions();
1108            return Ok(());
1109        }
1110        // A run that already lost its quorum never resumes into the verdict
1111        // machinery: `deliberate` and `vote` would otherwise clobber the
1112        // stalled marker back to Voting and the run would keep going past a
1113        // verdict that is no longer trustworthy. Everything already recorded is
1114        // kept, so the run stays resumable (or foldable) for a human to pick up.
1115        //
1116        // On --resume the run gets one chance to repair itself: the seats a
1117        // rate limit took out are re-asked. If their quota has since reset and
1118        // the quorum is restored, the run picks up and finishes; otherwise it
1119        // stays stale and still-resumable for a later retry. If it does not
1120        // recover, the returned status stays `Stalled` and nothing was
1121        // clobbered (the recovery only mutates entries for the lost seats).
1122        if self.state.status == RunStatus::Stalled {
1123            if self.recover_stall().await? {
1124                self.finish_after_tally().await?;
1125            } else {
1126                // Still below quorum: persist the marker and stay resumable.
1127                self.state.save()?;
1128            }
1129            return Ok(());
1130        }
1131        // A run parked inside `land` - watching CI, mid fix-round, or
1132        // waiting on the owner's merge approval - resumes directly into it,
1133        // never back through `prep`. Everything before `merge` already
1134        // concluded; that is the only way `status` reaches `Landing` in the
1135        // first place. Re-walking `review_loop` first would also be actively
1136        // wrong: its own status recomputation (see its doc) treats any
1137        // clean round as reason to set `status` to `Gating`, which would
1138        // clobber this marker before `merge` ever ran, and this run would
1139        // never find its way back into `land` at all.
1140        if self.state.status == RunStatus::Landing {
1141            self.run_land().await?;
1142            // `run_land` may have settled the run right here - CI came back
1143            // green and the PR merged, say - without ever passing back
1144            // through `merge`'s own trailing call. Whatever it left `status`
1145            // as is what this has to read.
1146            self.settle_questions();
1147            return Ok(());
1148        }
1149        // A run parked at `merge` on the owner's word about a withheld PR text
1150        // resumes straight into `merge`. Re-walking the tail would sync to the
1151        // base again (possibly rebasing, possibly spending the fixer) after the
1152        // gate had passed, and then push a tree nobody reviewed or gated.
1153        if self.state.github_text.is_some()
1154            && self.state.merge.is_none()
1155            && self.state.status == RunStatus::Gating
1156        {
1157            self.merge().await?;
1158            self.state.save()?;
1159            return Ok(());
1160        }
1161        self.prep().await?;
1162        if self.park_here()? {
1163            return Ok(());
1164        }
1165        self.advise().await?;
1166        if self.park_here()? {
1167            return Ok(());
1168        }
1169        self.implement().await?;
1170        if self.park_here()? {
1171            return Ok(());
1172        }
1173        // `after_implement` already saved the state and settled any open
1174        // questions when it set this; nothing later in the graph has
1175        // anything to judge.
1176        if self.state.status == RunStatus::VerifiedNoop {
1177            return Ok(());
1178        }
1179        self.judge().await?;
1180        if self.park_here()? {
1181            return Ok(());
1182        }
1183        self.deliberate().await?;
1184        if self.park_here()? {
1185            return Ok(());
1186        }
1187        self.vote().await?;
1188        if self.park_here()? {
1189            return Ok(());
1190        }
1191        self.tally()?;
1192        // A verdict that lost its quorum is not trustworthy: do not review,
1193        // gate, or merge on it. Everything already done is kept, so the run
1194        // stays resumable (or foldable); the human can replace the agent that
1195        // ran out of quota and pick it up.
1196        if self.state.status == RunStatus::Stalled {
1197            // Persist the stalled marker now — the normal end-of-execute save
1198            // below is below this early return, and without it a resumed run
1199            // would reload a pre-tally status and keep going.
1200            self.state.save()?;
1201            return Ok(());
1202        }
1203        self.finish_after_tally().await?;
1204        Ok(())
1205    }
1206
1207    /// A wave could not start its agents because a shared resource stayed busy
1208    /// (see [`RunState::resource_block`]): end the node without recording any
1209    /// result, leaving the run `Blocked` and resumable. No seat history,
1210    /// handover, quota loss or candidate state was written for it.
1211    fn stop_for_resource(&mut self) -> Result<()> {
1212        // A wait abandoned because the run is parking stays a park.
1213        if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1214        {
1215            self.state.status = RunStatus::Blocked;
1216        }
1217        self.state.save()
1218    }
1219
1220    /// Stop walking the graph if a node ended on a resource block.
1221    fn resource_halt(&mut self) -> Result<bool> {
1222        if self.state.resource_block.is_none() {
1223            return Ok(false);
1224        }
1225        self.stop_for_resource()?;
1226        Ok(true)
1227    }
1228
1229    /// Park here if asked to, recording it in the run's own timeline.
1230    ///
1231    /// Returns whether the caller should stop walking the graph. The state is
1232    /// saved either way by the node that just finished; this adds the event so
1233    /// the operator's card says why a run that is neither finished nor moving
1234    /// is sitting where it is.
1235    fn park_here(&mut self) -> Result<bool> {
1236        let parking = self.pause.parked() || self.interrupt.parked();
1237        if !parking && self.state.resource_block.is_some() {
1238            self.stop_for_resource()?;
1239            return Ok(true);
1240        }
1241        // Either handle asking is enough - see `Pause`'s own doc for why
1242        // they are never the same one. `interrupt` is checked second so a
1243        // reason it carries is preferred in the message below over a plain
1244        // shutdown park racing it at the same boundary.
1245        if !self.pause.parked() && !self.interrupt.parked() {
1246            return Ok(false);
1247        }
1248        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1249        let why = match cause.as_ref().map(ParkCause::label) {
1250            Some(reason) => format!(
1251                "parked after `{}` ({reason}) — resume to carry on from here",
1252                self.state.status.as_str()
1253            ),
1254            None => format!(
1255                "parked after `{}` — resume to carry on from here",
1256                self.state.status.as_str()
1257            ),
1258        };
1259        self.state.event("park", why);
1260        self.state
1261            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1262        self.state.parked = true;
1263        self.state.save()?;
1264        Ok(true)
1265    }
1266
1267    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1268    pub fn on_pause(&mut self, pause: Pause) {
1269        self.pause = pause;
1270    }
1271
1272    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1273    /// scheduler asking this one run - and no other - to park so a task
1274    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1275    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1276    /// handle.
1277    pub fn watch_interrupt(&mut self, pause: Pause) {
1278        self.interrupt = pause;
1279    }
1280
1281    /// Abandon this run's own open questions, once `status` has actually
1282    /// settled rather than merely paused.
1283    ///
1284    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1285    /// either back up with the candidates, the review round and the seat
1286    /// sessions already on disk, so a question an implementer asked mid-round
1287    /// may still get a real answer read by a real resume. Only the statuses
1288    /// `resumable` excludes are actually final: the run merged, it reached
1289    /// `Ready` with nothing left to do, it failed outright with no
1290    /// established point to continue from, or every candidate agreed, with
1291    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1292    /// every one of those the seat that asked is gone for good, exactly like
1293    /// the run being deleted under `magi run rm` - so the same cleanup
1294    /// applies, worded for what actually happened instead of "the run was
1295    /// deleted".
1296    ///
1297    /// Best-effort and silent on success: called from every place `status`
1298    /// can land on one of those three, including ones a resumed run revisits,
1299    /// so it must cost nothing when there was nothing open to begin with.
1300    fn settle_questions(&mut self) {
1301        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1302            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1303        }
1304    }
1305
1306    /// The tail of the graph after a trustworthy tally: fold losers, review,
1307    /// gate, merge, and persist.
1308    async fn finish_after_tally(&mut self) -> Result<()> {
1309        // Before the losers are folded: their branches are what the graft
1310        // reads from.
1311        self.graft().await?;
1312        if self.resource_halt()? {
1313            return Ok(());
1314        }
1315        self.fold_losers().await?;
1316        // Before review starts, and again right before the gate: a run's
1317        // review rounds can themselves take long enough for the base to move
1318        // a second time, and the gate is the one node whose "green" gets
1319        // acted on.
1320        self.sync_to_base().await?;
1321        if self.state.status == RunStatus::AlreadyInBase {
1322            return Ok(());
1323        }
1324        self.review_loop().await?;
1325        if self.resource_halt()? {
1326            return Ok(());
1327        }
1328        self.sync_to_base().await?;
1329        if self.state.status == RunStatus::AlreadyInBase {
1330            return Ok(());
1331        }
1332        self.gate().await?;
1333        self.merge().await?;
1334        self.state.save()?;
1335        Ok(())
1336    }
1337
1338    // ---------------------------------------------------------------- prep
1339
1340    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1341    /// run with the step and its output in the event; it is never swallowed.
1342    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1343    /// resume starts over instead of finding candidates and missing seats.
1344    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1345        let result = match ensure_setup_config(&mut self.state, repo).await {
1346            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1347            Err(e) => Err(e),
1348        };
1349        if let Err(e) = result {
1350            git::worktree_remove(repo, wt).await.ok();
1351            if in_prep {
1352                self.rollback_prep(repo).await;
1353            }
1354            self.state.status = RunStatus::Blocked;
1355            self.state.event(
1356                "setup",
1357                format!("worktree setup failed in {}: {e:#}", wt.display()),
1358            );
1359            self.state.save()?;
1360            return Err(e);
1361        }
1362        Ok(())
1363    }
1364
1365    /// Undo everything `prep` created, so a resume re-runs it whole.
1366    async fn rollback_prep(&mut self, repo: &Path) {
1367        let root = self.state.worktree_root();
1368        for c in std::mem::take(&mut self.state.candidates) {
1369            git::worktree_remove(repo, &c.worktree).await.ok();
1370            git::branch_delete(repo, &c.branch).await.ok();
1371        }
1372        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1373        for j in 1..=seats {
1374            git::worktree_remove(repo, &root.join(format!("judge-{j}")))
1375                .await
1376                .ok();
1377        }
1378        for k in 1..=self.state.config.graph.advisors {
1379            git::worktree_remove(repo, &root.join(format!("advisor-{k}")))
1380                .await
1381                .ok();
1382        }
1383    }
1384
1385    async fn prep(&mut self) -> Result<()> {
1386        if !self.state.candidates.is_empty() {
1387            return Ok(());
1388        }
1389        self.state.status = RunStatus::Prep;
1390        let repo = self.state.repo.clone();
1391        let base = self.state.base_commit.clone();
1392        let plan = refs::plan(&repo, &self.state.seeds).await?;
1393        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1394        let root = self.state.worktree_root();
1395        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1396
1397        // The hook is the write-time half of the blindness contract; the
1398        // presentation filter in `blind` is the half that cannot be bypassed.
1399        let hooks_dir = self.state.dir().join("hooks");
1400        if self.state.config.blind.commit_msg_hook {
1401            std::fs::create_dir_all(&hooks_dir)
1402                .with_context(|| format!("create {}", hooks_dir.display()))?;
1403            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1404            let path = hooks_dir.join("commit-msg");
1405            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1406            make_executable(&path)?;
1407            // Ref-counted rather than a plain idempotent set: with more than
1408            // one run able to be in flight in the same repository at once
1409            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1410            // true?" check cannot tell "another run of mine still needs
1411            // this" from "nobody does", and the run that happens to finish
1412            // first would disable the hook out from under a sibling still
1413            // relying on it.
1414            git::acquire_worktree_config(&repo).await?;
1415            self.state.enabled_worktree_config = true;
1416        }
1417
1418        // `[worktree] setup` hides its products per worktree, which needs the
1419        // same per-worktree config the hook does. Held until fold, like it.
1420        ensure_setup_config(&mut self.state, &repo).await?;
1421
1422        for (index, (spec, label)) in self
1423            .roles
1424            .implementers
1425            .clone()
1426            .into_iter()
1427            .zip(labels)
1428            .enumerate()
1429        {
1430            let branch = self.state.branch_for(label);
1431            let worktree = root.join(format!("cand-{label}"));
1432            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1433            if self.state.config.blind.commit_msg_hook {
1434                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1435            }
1436            git::local_exclude(&worktree, "/.magi/").await?;
1437            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1438                // Nothing of this prep may survive: a resume must start over,
1439                // not find half the candidates and skip the rest.
1440                git::worktree_remove(&repo, &worktree).await.ok();
1441                git::branch_delete(&repo, &branch).await.ok();
1442                self.rollback_prep(&repo).await;
1443                self.state.status = RunStatus::Blocked;
1444                self.state
1445                    .event("prep", format!("worktree setup failed: {e:#}"));
1446                self.state.save()?;
1447                return Err(e);
1448            }
1449            for pick in &plan.picks {
1450                if let Err(e) = git::cherry_pick(&worktree, pick).await {
1451                    self.state.status = RunStatus::Blocked;
1452                    self.state
1453                        .event("prep", format!("cannot apply referenced commit: {e}"));
1454                    self.state.save()?;
1455                    return Err(e);
1456                }
1457            }
1458            self.state.candidates.push(Candidate {
1459                index,
1460                label,
1461                agent: spec.id.clone(),
1462                branch,
1463                worktree,
1464                summary: String::new(),
1465                stat: String::new(),
1466                files: 0,
1467                commits: 0,
1468                empty: false,
1469                failed: None,
1470                verified_noop: None,
1471                duration_ms: 0,
1472                folded: false,
1473            });
1474        }
1475
1476        for j in 1..=self.roles.judges.len() {
1477            let wt = root.join(format!("judge-{j}"));
1478            if !wt.exists() {
1479                git::worktree_add_detached(&repo, &wt, &base).await?;
1480                self.setup_seat_worktree(&repo, &wt, true).await?;
1481            }
1482        }
1483
1484        // Disposable, detached checkouts for the design-deliberation stage's
1485        // advisor seats — the same shape as the judges' above, at the same
1486        // base commit, since advisors also only ever read. Sized off the
1487        // configured count directly rather than a resolved roster: unlike
1488        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1489        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1490        // `prep` has no `ResolvedRoles` field to read a count from here.
1491        if self.state.config.graph.advise {
1492            for k in 1..=self.state.config.graph.advisors {
1493                let wt = root.join(format!("advisor-{k}"));
1494                if !wt.exists() {
1495                    git::worktree_add_detached(&repo, &wt, &base).await?;
1496                    self.setup_seat_worktree(&repo, &wt, true).await?;
1497                }
1498            }
1499        }
1500
1501        // A judge cannot tell it is looking at its own patch — the seats keep
1502        // separate conversations — but a panel that shares agents with the
1503        // field is less independent than it looks, and that is worth saying out
1504        // loud once per run rather than leaving it in the config.
1505        let authors: Vec<&str> = self
1506            .roles
1507            .implementers
1508            .iter()
1509            .map(|a| a.id.as_str())
1510            .collect();
1511        let overlap: Vec<String> = self
1512            .roles
1513            .judges
1514            .iter()
1515            .enumerate()
1516            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1517            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1518            .collect();
1519        if !overlap.is_empty() {
1520            let note = format!(
1521                "{} also authored a candidate; blind, but the panel is less \
1522                 independent than {} distinct agents would be",
1523                overlap.join(", "),
1524                self.roles.judges.len()
1525            );
1526            self.state.event("prep", note);
1527        }
1528
1529        self.state.event(
1530            "prep",
1531            format!(
1532                "{} candidates, {} judges, base {} ({})",
1533                self.state.candidates.len(),
1534                self.roles.judges.len(),
1535                &self.state.base_commit[..7.min(self.state.base_commit.len())],
1536                self.state.base_branch
1537            ),
1538        );
1539        self.state.status = RunStatus::Implementing;
1540        self.state.save()?;
1541        Ok(())
1542    }
1543
1544    // -------------------------------------------------------------- advise
1545
1546    /// The design-deliberation stage: independent, read-only advisor seats
1547    /// each sketch a design before any implementer touches the repository,
1548    /// and (when at least one produced a usable proposal) a synthesis seat
1549    /// blends them into a brief `implement` carries in every candidate's
1550    /// prompt.
1551    ///
1552    /// `[graph] advise` is the on/off switch, on by default; `[graph]
1553    /// advisors` is the proposal count. Everything here is best-effort and
1554    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
1555    /// that cannot reach quota, or a synthesis seat that produced nothing
1556    /// usable all leave `implement` exactly as it was before this stage
1557    /// existed — the task instruction alone — rather than failing the whole
1558    /// competition over an enrichment stage. Every outcome is still recorded
1559    /// as an event, so a run that got nothing from this stage says why.
1560    ///
1561    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
1562    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
1563    /// resumed run whose stage failed would re-run it, and re-spend the
1564    /// agent calls, on every reentry before `implement`.
1565    ///
1566    /// Also skipped once any candidate shows implementation progress — the
1567    /// exact predicate `implement` itself uses to decide a candidate is no
1568    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
1569    /// is not enough: a run created by an older binary that predates this
1570    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
1571    /// an already-`Implementing`-or-later run under this build would
1572    /// otherwise walk straight back through `prep` (a no-op once candidates
1573    /// exist) into this node and spawn every advisor seat against worktrees
1574    /// `prep` never recreated — after implementation has already started,
1575    /// which is exactly the invariant this stage exists to guarantee.
1576    async fn advise(&mut self) -> Result<()> {
1577        let implement_untouched = self
1578            .state
1579            .candidates
1580            .iter()
1581            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1582        if !self.state.config.graph.advise || self.state.advise_attempted {
1583            return Ok(());
1584        }
1585        if !implement_untouched {
1586            self.state.event(
1587                "advise",
1588                "skipping the design-deliberation stage: at least one \
1589                 candidate already shows implementation progress, so this \
1590                 run is past the point the stage exists to run before"
1591                    .to_owned(),
1592            );
1593            self.state.advise_attempted = true;
1594            self.state.save()?;
1595            return Ok(());
1596        }
1597        let run_id = self.state.id.clone();
1598        let prompts = self.state.config.prompts.clone();
1599        let instruction = self.state.instruction.clone();
1600        let language = self.state.config.graph.language.clone();
1601        let root = self.state.worktree_root();
1602        let n = self.state.config.graph.advisors;
1603        let where_recorded = self.state.dir().join("run.json");
1604
1605        let seats = match self.state.config.advisors() {
1606            Ok(seats) if !seats.is_empty() => seats,
1607            Ok(_) => {
1608                self.state.event(
1609                    "advise",
1610                    format!(
1611                        "[graph] advisors is 0; skipping the design-deliberation \
1612                         stage and continuing without a synthesis brief (see {})",
1613                        where_recorded.display()
1614                    ),
1615                );
1616                self.state.advise_attempted = true;
1617                self.state.save()?;
1618                return Ok(());
1619            }
1620            Err(e) => {
1621                self.state.event(
1622                    "advise",
1623                    format!(
1624                        "could not resolve advisor seats ({e:#}); continuing \
1625                         without a design-deliberation brief (see {})",
1626                        where_recorded.display()
1627                    ),
1628                );
1629                self.state.advise_attempted = true;
1630                self.state.save()?;
1631                return Ok(());
1632            }
1633        };
1634
1635        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1636        let artifacts = agent::artifacts_dir(&self.state.dir());
1637        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1638
1639        let mut jobs = Vec::new();
1640        for (i, spec) in seats.iter().cloned().enumerate() {
1641            let seat_key = format!("advisor-{}", i + 1);
1642            let seat = self.seat(&seat_key, &spec.id);
1643            jobs.push(SeatJob {
1644                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1645                spec,
1646                seat,
1647                cwd: worktrees[i % worktrees.len()].clone(),
1648                timeout,
1649                allow_write: false,
1650                sessions: false,
1651                artifacts: artifacts.clone(),
1652                stem: seat_key,
1653                handover: None,
1654            });
1655        }
1656
1657        self.state.event(
1658            "advise",
1659            format!(
1660                "{} advisor seat(s) sketching a design in parallel",
1661                jobs.len()
1662            ),
1663        );
1664        let mut quota_losses = Vec::new();
1665        let cache = self.state.config.cache_dir();
1666        let ctx = WaveCtx {
1667            carry_seats: false,
1668            pauses: Some((&self.pause, &self.interrupt)),
1669            run: &run_id,
1670            node: "advise",
1671            prompts: &prompts,
1672            cache: cache.as_deref(),
1673            round: None,
1674        };
1675        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
1676        let results = ask_json_wave::<Proposal>(
1677            jobs,
1678            Arc::clone(&self.sem),
1679            self.state.config.graph.retries,
1680            &advisor_roster,
1681            &ctx,
1682            &mut quota_losses,
1683            &mut self.state,
1684            &|p: &Proposal| p.validate(),
1685        )
1686        .await;
1687        if self.state.resource_block.is_some() {
1688            return self.stop_for_resource();
1689        }
1690        self.state.quota.extend(quota_losses);
1691
1692        let mut records = Vec::with_capacity(results.len());
1693        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1694            let agent_id = seat.agent.clone();
1695            self.state.seats.insert(seat.key.clone(), seat);
1696            match res {
1697                Ok((proposal, out)) => {
1698                    self.state
1699                        .event("advise", format!("advisor-{} proposed a design", i + 1));
1700                    records.push(advise::AdvisorRecord::proposed(
1701                        i + 1,
1702                        agent_id,
1703                        proposal,
1704                        out.duration_ms,
1705                    ));
1706                }
1707                Err(e) => {
1708                    self.state.event(
1709                        "advise",
1710                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1711                    );
1712                    records.push(advise::AdvisorRecord::failed(
1713                        i + 1,
1714                        agent_id,
1715                        e.to_string(),
1716                    ));
1717                }
1718            }
1719        }
1720
1721        let mut advice = advise::Advice {
1722            records,
1723            synthesis: None,
1724        };
1725        if advice.proposals().is_empty() {
1726            self.state.event(
1727                "advise",
1728                "no advisor produced a usable proposal; continuing without a \
1729                 synthesis brief"
1730                    .to_owned(),
1731            );
1732        } else {
1733            match self
1734                .synthesize_brief(
1735                    &advice,
1736                    &instruction,
1737                    &language,
1738                    &worktrees[0],
1739                    &artifacts,
1740                    &run_id,
1741                    &prompts,
1742                    cache.as_deref(),
1743                )
1744                .await
1745            {
1746                Ok(Some(text)) => {
1747                    self.state.event(
1748                        "advise",
1749                        "synthesized a design brief for the implementer".to_owned(),
1750                    );
1751                    advice.synthesis = Some(text);
1752                }
1753                Ok(None) => {
1754                    self.state.event(
1755                        "advise",
1756                        "the synthesis seat produced nothing usable; continuing \
1757                         without a design brief"
1758                            .to_owned(),
1759                    );
1760                }
1761                Err(e) => {
1762                    self.state.event(
1763                        "advise",
1764                        format!("could not synthesize a design brief: {e:#}"),
1765                    );
1766                }
1767            }
1768        }
1769        advise::apply_reflection(&mut advice);
1770
1771        self.state.advice = Some(advice);
1772        self.state.advise_attempted = true;
1773        self.state.save()?;
1774        Ok(())
1775    }
1776
1777    /// The synthesis seat: reads every advisor's proposal and blends them
1778    /// into the design brief `advise` stores on [`RunState::advice`]. Split
1779    /// out of [`Runner::advise`] only for readability — it is not called
1780    /// anywhere else.
1781    ///
1782    /// Picked the same way [`crate::talk`]'s standing conversation and
1783    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
1784    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
1785    /// order (a claude seat, else the first runnable agent in roster order)
1786    /// used when that field is unset — see `[roles] synthesizer`'s own doc
1787    /// in [`crate::config`] for why a dedicated field exists here at all.
1788    #[allow(clippy::too_many_arguments)]
1789    async fn synthesize_brief(
1790        &mut self,
1791        advice: &advise::Advice,
1792        instruction: &str,
1793        language: &str,
1794        cwd: &Path,
1795        artifacts: &Path,
1796        run_id: &str,
1797        prompts: &Prompts,
1798        cache: Option<&Path>,
1799    ) -> Result<Option<String>> {
1800        let chain = agent::pick_chain(
1801            &self.state.config.agents,
1802            self.state.config.roles.synthesizer.as_ref(),
1803            &agent::installed,
1804            "synthesizer",
1805        )?;
1806        let proposals = advice.proposals();
1807        let mut prompt = prompt::with_overlay(
1808            prompt::synthesize_brief(instruction, &proposals, language),
1809            prompts.overlay("advise"),
1810        );
1811        if cache.is_some() {
1812            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
1813            // below — see `prompt::build_cache_note`'s doc for why telling a
1814            // read-only seat to build through the shared cache is exactly how
1815            // a sandbox's write refusal gets misread as a defect.
1816            prompt.push('\n');
1817            prompt.push_str(&prompt::build_cache_note("advise", false));
1818        }
1819        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1820        // Each id is tried once, in order; a quota hit, error or unusable
1821        // answer moves to the next. The seat is single-turn (`sessions:
1822        // false`) and the prompt is the whole context, so a fallback agent
1823        // needs nothing carried over.
1824        let mut last = None;
1825        for (n, spec) in chain.iter().enumerate() {
1826            if n > 0 {
1827                self.state
1828                    .event("advise", format!("synthesis falling back to {}", spec.id));
1829            }
1830            let mut seat = self.seat("advise-synthesis", &spec.id);
1831            let outcome = agent::invoke(
1832                spec,
1833                &mut seat,
1834                &Invocation {
1835                    cwd,
1836                    prompt: &prompt,
1837                    timeout,
1838                    allow_write: false,
1839                    unsandboxed: false,
1840                    sessions: false,
1841                    artifacts,
1842                    stem: &if n == 0 {
1843                        "advise-synthesis".to_owned()
1844                    } else {
1845                        format!("advise-synthesis-{}", spec.id)
1846                    },
1847                    run: run_id,
1848                    node: "advise",
1849                    cache_dir: None,
1850                    attachments: &[],
1851                    writable: &[],
1852                },
1853            )
1854            .await;
1855            if outcome.is_ok() {
1856                self.state.seats.insert(seat.key.clone(), seat);
1857            }
1858            let advance = agent::chain_advances(&outcome);
1859            last = Some(outcome);
1860            if !advance {
1861                break;
1862            }
1863        }
1864        // Exhausted: the last attempt's result is what a single failed seat
1865        // would have produced.
1866        let out = last.expect("a chain holds at least one agent")?;
1867        if !out.usable() {
1868            return Ok(None);
1869        }
1870        let text =
1871            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1872        Ok((!text.trim().is_empty()).then_some(text))
1873    }
1874
1875    // ----------------------------------------------------------- implement
1876
1877    async fn implement(&mut self) -> Result<()> {
1878        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1879        // agent files with `magi task add` name the run that paid for it. The
1880        // prompt overlay is cloned alongside it because the waves borrow it
1881        // while `self` is mutably borrowed by the node's own bookkeeping.
1882        let run_id = self.state.id.clone();
1883        let prompts = self.state.config.prompts.clone();
1884        let todo: Vec<usize> = self
1885            .state
1886            .candidates
1887            .iter()
1888            .enumerate()
1889            .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1890            .map(|(i, _)| i)
1891            .collect();
1892        if todo.is_empty() {
1893            return self.after_implement();
1894        }
1895        self.state.status = RunStatus::Implementing;
1896
1897        let language = self.state.config.graph.language.clone();
1898        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1899        let sessions = self.state.config.graph.sessions;
1900        let artifacts = agent::artifacts_dir(&self.state.dir());
1901        // The design-deliberation stage's blended brief, when `advise` found
1902        // one — carried into every implementer's prompt the same way
1903        // regardless of which candidate it is.
1904        let brief = self
1905            .state
1906            .advice
1907            .as_ref()
1908            .and_then(|a| a.synthesis.as_deref())
1909            .map(str::to_owned);
1910        let attachments = self.state.attachments.clone();
1911
1912        let mut jobs = Vec::new();
1913        for &i in &todo {
1914            let (index, label, worktree) = {
1915                let c = &self.state.candidates[i];
1916                (c.index, c.label, c.worktree.clone())
1917            };
1918            let spec = self.roles.implementers[index].clone();
1919            let seat_key = format!("impl-{label}");
1920            let seat = self.seat(&seat_key, &spec.id);
1921            let instruction = seeded_instruction(&self.state);
1922            jobs.push(SeatJob {
1923                spec,
1924                seat,
1925                prompt: prompt::implement(
1926                    &instruction,
1927                    &worktree.to_string_lossy(),
1928                    &language,
1929                    brief.as_deref(),
1930                    &attachments,
1931                ),
1932                cwd: worktree,
1933                timeout,
1934                allow_write: true,
1935                sessions,
1936                artifacts: artifacts.clone(),
1937                stem: format!("impl-{label}"),
1938                handover: None,
1939            });
1940        }
1941
1942        self.state.event(
1943            "implement",
1944            format!("{} candidates in parallel", jobs.len()),
1945        );
1946        // Kept so a seat whose CLI hung up can be asked again from the same
1947        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
1948        // can update a seat's own entry once a fallback agent takes it over —
1949        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
1950        // whichever agent actually answered, not the one that quota'd out.
1951        let mut sent = jobs.clone();
1952        let cache = self.state.config.cache_dir();
1953        let ctx = WaveCtx {
1954            carry_seats: false,
1955            pauses: Some((&self.pause, &self.interrupt)),
1956            run: &run_id,
1957            node: "implement",
1958            prompts: &prompts,
1959            cache: cache.as_deref(),
1960            round: None,
1961        };
1962        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1963        if wave_blocked(&results) {
1964            return self.stop_for_resource();
1965        }
1966        self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1967            .await;
1968        self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
1969            .await;
1970        self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1971            .await;
1972        if self.state.resource_block.is_some() {
1973            return self.stop_for_resource();
1974        }
1975
1976        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1977            let seat_key = seat.key.clone();
1978            // A quota fallback (`resume_seat_handovers`) may have handed this
1979            // seat to a different agent than the one `prep` recorded on the
1980            // candidate; the stats tables and any later fixer-defaults-to-
1981            // winner's-author lookup must credit whoever actually answered —
1982            // unless every fallback also quota'd out, in which case nobody
1983            // actually answered and crediting the last agent tried would
1984            // erase every earlier agent's own quota loss from the stats
1985            // tables instead of just this one seat's.
1986            let agent = seat.agent.clone();
1987            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
1988            self.state.seats.insert(seat.key.clone(), seat);
1989            let label = self.state.candidates[i].label;
1990            let worktree = self.state.candidates[i].worktree.clone();
1991            let base = self.state.base_commit.clone();
1992
1993            let (summary, duration, failed, verified_claim) = match out {
1994                AgentOutcome::Ok(o) => {
1995                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1996                    let failed = (!o.usable()).then(|| {
1997                        if o.timed_out {
1998                            "agent timed out".to_owned()
1999                        } else {
2000                            format!("agent exited with {:?}", o.exit_code)
2001                        }
2002                    });
2003                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2004                    (text, o.duration_ms, failed, verified_claim)
2005                }
2006                // Left un-resumed by `resume_undelivered` (a dirty tree
2007                // already rescues the work, or there was no session left to
2008                // resume into) — reported like the ordinary failure it is,
2009                // never as if `o.text` (the CLI's raw error JSON) were an
2010                // answer.
2011                AgentOutcome::Dropped(o) => {
2012                    let why = o
2013                        .dropped
2014                        .as_ref()
2015                        .map(|d| d.why.as_str())
2016                        .unwrap_or("the CLI ended the stream without delivering its answer");
2017                    (
2018                        String::new(),
2019                        o.duration_ms,
2020                        Some(format!("the CLI dropped the stream ({why})")),
2021                        None,
2022                    )
2023                }
2024                AgentOutcome::Quota(o) => {
2025                    self.state.quota.push(QuotaLoss {
2026                        seat: seat_key,
2027                        node: "implement".to_owned(),
2028                        at: Timestamp::now(),
2029                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2030                    });
2031                    (
2032                        String::new(),
2033                        o.duration_ms,
2034                        Some("rate limited (quota); produced no change".to_owned()),
2035                        None,
2036                    )
2037                }
2038                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2039                    (String::new(), 0, Some(e), None)
2040                }
2041            };
2042
2043            // Rescue anything the agent edited but never committed: an
2044            // uncommitted candidate would silently be an empty one.
2045            let rescued = match git::rescue_commit(
2046                &worktree,
2047                &format!("magi: candidate {label} (uncommitted work)"),
2048            )
2049            .await
2050            {
2051                Ok(r) => {
2052                    self.state.note_withheld("implement", &r.withheld);
2053                    r.committed
2054                }
2055                Err(_) => false,
2056            };
2057            let commits = git::commits_ahead(&worktree, &base, "HEAD")
2058                .await
2059                .unwrap_or(0);
2060            let patch = git::diff(&worktree, &base, "HEAD")
2061                .await
2062                .unwrap_or_default();
2063            let stat = git::diff_stat(&worktree, &base, "HEAD")
2064                .await
2065                .unwrap_or_default();
2066            let files = git::changed_files(&worktree, &base, "HEAD")
2067                .await
2068                .map(|f| f.len())
2069                .unwrap_or(0);
2070            write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2071
2072            let c = &mut self.state.candidates[i];
2073            if !exhausted_the_fallback_chain {
2074                c.agent = agent;
2075            }
2076            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2077            c.stat = stat;
2078            c.files = files;
2079            c.commits = commits;
2080            c.duration_ms = duration;
2081            c.empty = commits == 0 || patch.trim().is_empty();
2082            // An agent that failed but still produced a committed change stays
2083            // in the running: the patch is what gets judged, not the exit code.
2084            c.failed = match failed {
2085                Some(_) if c.empty => failed,
2086                _ => None,
2087            };
2088            // Only an empty candidate can be a verified no-op: a claim next
2089            // to a real patch is not what the marker is for, and `c.failed`
2090            // being `Some` here already implies `verified_claim` was never
2091            // set (see the guard above the match that produced it).
2092            c.verified_noop = if c.empty { verified_claim } else { None };
2093            let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
2094                (Some(e), _, _, _) => format!("candidate {label}: {e}"),
2095                (None, true, Some(_), _) => {
2096                    format!("candidate {label}: no change produced (agent-verified no-op)")
2097                }
2098                (None, true, None, _) => format!("candidate {label}: no change produced"),
2099                (None, false, _, true) => {
2100                    format!(
2101                        "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
2102                    )
2103                }
2104                (None, false, _, false) => {
2105                    format!("candidate {label}: {files} files, {commits} commits")
2106                }
2107            };
2108            self.state.event("implement", note);
2109            self.state.save()?;
2110        }
2111
2112        self.after_implement()
2113    }
2114
2115    /// Ask again, once, for work a CLI did and then failed to hand over.
2116    ///
2117    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2118    /// status with an empty response and a usage report showing output tokens,
2119    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2120    /// seven minutes and 14,267 output tokens that arrived as an empty
2121    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2122    ///
2123    /// Two conditions, and both matter:
2124    ///
2125    /// - **Only when the tree is untouched.** Often the agent has already
2126    ///   written its files and only the closing message was lost; the rescue
2127    ///   commit below picks that up and there is nothing to ask for. Re-asking
2128    ///   then would pay for a second implementation of work already on disk.
2129    /// - **Once.** A CLI that drops one stream can drop the next, and this
2130    ///   node is the most expensive in the graph.
2131    ///
2132    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2133    /// dropped reply still carried its `conversation_id`, so the seat is asked
2134    /// to finish what it was doing rather than sent the whole task again. It
2135    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2136    /// node's - for the same reason a re-ranked judge does: restating finished
2137    /// work is not the work.
2138    ///
2139    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2140    /// same way until it resets, while an abandoned conversation is still
2141    /// there to be picked up.
2142    async fn resume_undelivered(
2143        &mut self,
2144        results: &mut [(usize, SeatState, AgentOutcome)],
2145        sent: &[SeatJob],
2146        prompts: &Prompts,
2147        run_id: &str,
2148    ) {
2149        for (wi, seat, out) in results.iter_mut() {
2150            let Some(dropped) = (match &*out {
2151                AgentOutcome::Dropped(o) => o.dropped.clone(),
2152                _ => None,
2153            }) else {
2154                continue;
2155            };
2156            let Some(job) = sent.get(*wi) else { continue };
2157            // Already on disk? Then only the closing message was lost.
2158            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2159                self.state.event(
2160                    "implement",
2161                    format!(
2162                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2163                         work is in the tree",
2164                        seat.key, dropped.output_tokens, dropped.why
2165                    ),
2166                );
2167                continue;
2168            }
2169            // The re-ask only makes sense as a resume: `resume_after_drop`
2170            // says nothing about the task, trusting the seat to still hold it.
2171            // Without a session to resume — sessions disabled, or this CLI's
2172            // drop shape happened not to carry a session id — that prompt
2173            // would open a brand-new conversation with no context at all,
2174            // which is worse than leaving this as the ordinary failure it
2175            // already is.
2176            if !has_context(&job.spec, seat, job.sessions) {
2177                self.state.event(
2178                    "implement",
2179                    format!(
2180                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2181                         is no session left to resume",
2182                        seat.key, dropped.output_tokens, dropped.why
2183                    ),
2184                );
2185                continue;
2186            }
2187            self.state.event(
2188                "implement",
2189                format!(
2190                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2191                     conversation",
2192                    seat.key, dropped.output_tokens, dropped.why
2193                ),
2194            );
2195            let mut retry = job.clone();
2196            retry.seat = seat.clone();
2197            retry.prompt = prompt::resume_after_drop(&dropped.why);
2198            retry.timeout = retry_budget(job.timeout, true);
2199            retry.stem = format!("{}-resume", job.stem);
2200            let cache = self.state.config.cache_dir();
2201            let ctx = WaveCtx {
2202                carry_seats: false,
2203                pauses: Some((&self.pause, &self.interrupt)),
2204                run: run_id,
2205                node: "implement",
2206                prompts,
2207                cache: cache.as_deref(),
2208                round: None,
2209            };
2210            let (resumed_seat, resumed) =
2211                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2212            *seat = resumed_seat;
2213            *out = resumed;
2214        }
2215    }
2216
2217    /// Fall an implement seat through to the next untried agent in the
2218    /// implementer roster when it lost to quota — or, since the handover was
2219    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2220    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2221    /// quota path itself being unchanged — instead of leaving the
2222    /// seat's loss final the moment one agent's account runs dry.
2223    ///
2224    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2225    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2226    /// truncates to the single slot rotation picked, so a solo task whose one
2227    /// implementer hits quota mid-run used to have nothing else to try. This
2228    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2229    /// unrotated roster — which is the only place the *other* candidates in
2230    /// the machine's roster still exist once `implementers` has been cut down
2231    /// to size.
2232    ///
2233    /// Walks forward from just past the seat's own original position in the
2234    /// roster, never wrapping back to the front: a later candidate slot (say
2235    /// `beta`, the roster's second entry) must fall through to the *next*
2236    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2237    /// almost certainly a different candidate's own agent already — and once
2238    /// the roster's tail is exhausted there is nothing left to fall through
2239    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2240    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2241    /// let this retry that id forever. The loop keeps falling through until
2242    /// an attempt lands something other than `Quota` or the roster's tail
2243    /// runs out of untried ids, at which point the seat is left exactly as
2244    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2245    /// `QuotaLoss` recorded, the candidate failed/empty.
2246    ///
2247    /// `sent` is taken mutably and updated with the fallback agent's spec:
2248    /// `resume_unconfirmed_commands`, which runs after this and also reads
2249    /// `sent`, must see whichever agent actually ended up answering the seat
2250    /// — reading the stale, original spec there would check session
2251    /// eligibility against the wrong CLI and could hand a fallback agent's
2252    /// session id to the agent that just lost the seat to quota.
2253    ///
2254    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2255    /// — `self.seat` only reuses state when the agent id is unchanged, so
2256    /// handing it a different id already gets this for free. Reusing the old
2257    /// seat would resume a different CLI's session as if it were a
2258    /// continuation of this one.
2259    ///
2260    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2261    /// a quota loss cuts an agent off mid-turn, so anything already in the
2262    /// tree is unfinished work, not a completed candidate a re-ask would pay
2263    /// for twice. A dirty tree is rescued into a commit first (the same
2264    /// neutral-identity rescue `implement`'s own outcome loop gives every
2265    /// candidate) so the next agent starts clean.
2266    ///
2267    /// The new agent gets the implementer's full prompt and full
2268    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2269    /// it has no session and no context, and is implementing the task from
2270    /// nothing, unlike a resumed drop which is only restating work already
2271    /// done.
2272    ///
2273    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2274    /// `implement` event, never into `self.state.quota` — that is what
2275    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2276    /// go unspent, and a seat that ultimately recovered on its second or
2277    /// third agent is not the stalled panel that check exists to catch. Only
2278    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2279    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2280    /// outcome loop already has — this helper never pushes to it itself.
2281    async fn resume_seat_handovers(
2282        &mut self,
2283        results: &mut [(usize, SeatState, AgentOutcome)],
2284        sent: &mut [SeatJob],
2285        prompts: &Prompts,
2286        run_id: &str,
2287    ) {
2288        let instruction = seeded_instruction(&self.state);
2289        let language = self.state.config.graph.language.clone();
2290        let brief = self
2291            .state
2292            .advice
2293            .as_ref()
2294            .and_then(|a| a.synthesis.as_deref())
2295            .map(str::to_owned);
2296        let attachments = self.state.attachments.clone();
2297        for (wi, seat, out) in results.iter_mut() {
2298            // Who holds the other candidate seats of this wave right now
2299            // (earlier handovers already written back to `sent`).
2300            let others: BTreeSet<String> = sent
2301                .iter()
2302                .enumerate()
2303                .filter(|(j, _)| j != wi)
2304                .map(|(_, j)| j.spec.id.clone())
2305                .collect();
2306            let Some(job) = sent.get_mut(*wi) else {
2307                continue;
2308            };
2309            // Where the seat's own original agent sits in the roster — the
2310            // fallback walk starts just past here, never at the front, so a
2311            // later candidate slot's quota loss does not fall back onto an
2312            // earlier slot's own agent.
2313            let start = self
2314                .roles
2315                .implementer_roster
2316                .iter()
2317                .position(|s| s.id == job.spec.id)
2318                .unwrap_or(0);
2319            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2320            let mut fallback_attempt = 0usize;
2321            let mut prev: Option<FailClass> = None;
2322            while let Some(cur) = FailClass::of(&*out) {
2323                if !should_hand_over(prev.as_ref(), &cur) {
2324                    break;
2325                }
2326                let Some(next) =
2327                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2328                        .cloned()
2329                else {
2330                    break;
2331                };
2332                tried.insert(next.id.clone());
2333                fallback_attempt += 1;
2334
2335                if let Ok(r) = git::rescue_commit(
2336                    &job.cwd,
2337                    &format!(
2338                        "magi: candidate {} (uncommitted work before {} fallback)",
2339                        seat.key,
2340                        if cur == FailClass::Quota {
2341                            "quota"
2342                        } else {
2343                            "handover"
2344                        }
2345                    ),
2346                )
2347                .await
2348                {
2349                    self.state.note_withheld("implement", &r.withheld);
2350                }
2351
2352                record_handover(
2353                    &mut self.state,
2354                    "implement",
2355                    &seat.key,
2356                    &seat.agent,
2357                    &next.id,
2358                    &cur,
2359                    &fail_reason(&*out),
2360                );
2361                prev = Some(cur.clone());
2362
2363                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2364                self.state.seats.insert(seat.key.clone(), new_seat.clone());
2365                // Kept in sync on `sent` itself, not just the local retry: a
2366                // later helper (`resume_unconfirmed_commands`) reads `sent`
2367                // after this one returns and must see whichever agent is now
2368                // occupying the seat, not the one that just quota'd out —
2369                // otherwise it would judge session/continuation eligibility
2370                // by the wrong CLI and could resend a fallback's session id
2371                // to the agent that lost it the seat in the first place.
2372                job.spec = next.clone();
2373                let mut retry = job.clone();
2374                retry.seat = new_seat;
2375                retry.prompt = prompt::implement(
2376                    &instruction,
2377                    &job.cwd.to_string_lossy(),
2378                    &language,
2379                    brief.as_deref(),
2380                    &attachments,
2381                );
2382                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2383                let cache = self.state.config.cache_dir();
2384                let ctx = WaveCtx {
2385                    carry_seats: false,
2386                    pauses: Some((&self.pause, &self.interrupt)),
2387                    run: run_id,
2388                    node: "implement",
2389                    prompts,
2390                    cache: cache.as_deref(),
2391                    round: None,
2392                };
2393                let (fallback_seat, fallback_out) = run_one(
2394                    retry,
2395                    Arc::clone(&self.sem),
2396                    &ctx,
2397                    &mut self.state,
2398                    fallback_attempt,
2399                )
2400                .await;
2401                *seat = fallback_seat;
2402                *out = fallback_out;
2403            }
2404        }
2405    }
2406
2407    /// Ask an implement seat's own CLI to confirm what it started, once, when
2408    /// its reply reported a command whose completion status it never
2409    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
2410    /// that does and does not mean.
2411    ///
2412    /// The completion contract this task asks for, extended to `implement`
2413    /// with the same signal `continue_fix_report` reads for the fixer,
2414    /// rather than a keyword search over the reply or a hard requirement on
2415    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
2416    /// a candidate's CLI turn ended cleanly while a test run it had started
2417    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
2418    /// named in it at all is untouched by this: `commands` is empty, so
2419    /// there is nothing to be unconfirmed.
2420    ///
2421    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
2422    /// this is not about recovering edits that might already be on disk, it
2423    /// is about a result the seat itself never vouched for, which resuming
2424    /// asks for regardless of what the tree already holds. Bounded to one
2425    /// attempt for the same reason `resume_undelivered` is — this is the
2426    /// most expensive node in the graph — and a seat that still cannot
2427    /// confirm on that attempt is left as whatever its (possibly still
2428    /// unconfirmed) reply says; this does not invent a new "failed" reason
2429    /// for a candidate that otherwise produced a real, committed change.
2430    async fn resume_unconfirmed_commands(
2431        &mut self,
2432        results: &mut [(usize, SeatState, AgentOutcome)],
2433        sent: &[SeatJob],
2434        prompts: &Prompts,
2435        run_id: &str,
2436    ) {
2437        for (wi, seat, out) in results.iter_mut() {
2438            let AgentOutcome::Ok(o) = &*out else {
2439                continue;
2440            };
2441            if !has_unconfirmed_command(&o.commands) {
2442                continue;
2443            }
2444            let Some(job) = sent.get(*wi) else { continue };
2445            if !has_context(&job.spec, seat, job.sessions) {
2446                self.state.event(
2447                    "implement",
2448                    format!(
2449                        "{}: the reply named a command whose own CLI never confirmed the exit \
2450                         status of, but there is no session left to resume",
2451                        seat.key
2452                    ),
2453                );
2454                continue;
2455            }
2456            self.state.event(
2457                "implement",
2458                format!(
2459                    "{}: the reply named a command whose own CLI never confirmed the exit \
2460                     status of; resuming the conversation",
2461                    seat.key
2462                ),
2463            );
2464            let mut retry = job.clone();
2465            retry.seat = seat.clone();
2466            retry.prompt = prompt::resume_incomplete(
2467                "a command in your last reply had no confirmed exit status",
2468            );
2469            retry.timeout = retry_budget(job.timeout, true);
2470            retry.stem = format!("{}-confirm", job.stem);
2471            let cache = self.state.config.cache_dir();
2472            let ctx = WaveCtx {
2473                carry_seats: false,
2474                pauses: Some((&self.pause, &self.interrupt)),
2475                run: run_id,
2476                node: "implement",
2477                prompts,
2478                cache: cache.as_deref(),
2479                round: None,
2480            };
2481            let (resumed_seat, resumed) =
2482                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2483            *seat = resumed_seat;
2484            *out = resumed;
2485        }
2486    }
2487
2488    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
2489    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
2490    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
2491    /// that motivated this.
2492    ///
2493    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
2494    /// own nudge loop already covers for judge/review/vote seats, and not a
2495    /// dropped stream, which [`Runner::resume_undelivered`] covers for
2496    /// implement seats: here the CLI turn genuinely finished while the node's
2497    /// own work — the fixer's account of what it did — had not. Gated purely
2498    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
2499    /// reply, never on any wording in it, so a fixer whose valid, first-try
2500    /// `FixReport` happens to mention having waited on a background test is
2501    /// never resumed — the `Ok(report)` branch at the call site returns
2502    /// before this is ever invoked.
2503    ///
2504    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
2505    /// attempt ([`retry_budget`]), nothing attempted once the session is
2506    /// gone, and a quota hit ends the loop immediately rather than retrying a
2507    /// rate limit that fails the same way again.
2508    async fn continue_fix_report(
2509        &mut self,
2510        mut seat: SeatState,
2511        parse_err: String,
2512        job: &SeatJob,
2513        prompts: &Prompts,
2514        run_id: &str,
2515        round: usize,
2516    ) -> (
2517        SeatState,
2518        Option<FixReport>,
2519        Option<String>,
2520        ContinuationRecord,
2521    ) {
2522        let mut last_err = parse_err;
2523        let mut cumulative_wait_ms = 0u64;
2524        let mut attempts = 0usize;
2525        loop {
2526            if !has_context(&job.spec, &seat, job.sessions) {
2527                self.state.event(
2528                    "fix",
2529                    format!(
2530                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
2531                         session left to resume into"
2532                    ),
2533                );
2534                let outcome = if attempts == 0 {
2535                    ContinuationOutcome::NoSession
2536                } else {
2537                    ContinuationOutcome::Exhausted
2538                };
2539                return (
2540                    seat,
2541                    None,
2542                    Some(format!("unparsable fix report: {last_err}")),
2543                    ContinuationRecord {
2544                        attempts,
2545                        cumulative_wait_ms,
2546                        outcome,
2547                    },
2548                );
2549            }
2550            if attempts >= MAX_FIX_CONTINUATIONS {
2551                self.state.event(
2552                    "fix",
2553                    format!(
2554                        "round {round}: fixer's reply still had no adoption report after \
2555                         {attempts} continuation(s) ({last_err}); giving up"
2556                    ),
2557                );
2558                return (
2559                    seat,
2560                    None,
2561                    Some(format!(
2562                        "unparsable fix report after {attempts} continuation(s): {last_err}"
2563                    )),
2564                    ContinuationRecord {
2565                        attempts,
2566                        cumulative_wait_ms,
2567                        outcome: ContinuationOutcome::Exhausted,
2568                    },
2569                );
2570            }
2571            attempts += 1;
2572            self.state.event(
2573                "fix",
2574                format!(
2575                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
2576                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
2577                ),
2578            );
2579            let mut retry = job.clone();
2580            retry.seat = seat.clone();
2581            retry.prompt = prompt::resume_incomplete(&last_err);
2582            retry.timeout = retry_budget(job.timeout, true);
2583            retry.stem = format!("{}-continue{attempts}", job.stem);
2584            let cache = self.state.config.cache_dir();
2585            let ctx = WaveCtx {
2586                carry_seats: false,
2587                pauses: Some((&self.pause, &self.interrupt)),
2588                run: run_id,
2589                node: "fix",
2590                prompts,
2591                cache: cache.as_deref(),
2592                round: Some(round),
2593            };
2594            let (resumed_seat, resumed_out) = run_one(
2595                retry,
2596                Arc::clone(&self.sem),
2597                &ctx,
2598                &mut self.state,
2599                attempts,
2600            )
2601            .await;
2602            seat = resumed_seat;
2603            match resumed_out {
2604                AgentOutcome::Ok(o) => {
2605                    cumulative_wait_ms += o.duration_ms;
2606                    match verdict::extract_json::<FixReport>(&o.text) {
2607                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
2608                            self.state.event(
2609                                "fix",
2610                                format!(
2611                                    "round {round}: fixer's adoption report recovered after \
2612                                     {attempts} continuation(s)"
2613                                ),
2614                            );
2615                            return (
2616                                seat,
2617                                Some(report),
2618                                None,
2619                                ContinuationRecord {
2620                                    attempts,
2621                                    cumulative_wait_ms,
2622                                    outcome: ContinuationOutcome::Resumed,
2623                                },
2624                            );
2625                        }
2626                        // The report parsed, but this same reply's own
2627                        // CommandEvidence — the identical record `state.jobs`
2628                        // renders — names a command whose CLI never
2629                        // confirmed an exit status. Read together, that is
2630                        // not a resolved answer: keep nudging rather than
2631                        // accept a report standing next to a command the
2632                        // seat's own CLI cannot vouch for.
2633                        Ok(_) => {
2634                            last_err = "the reply parsed, but it reported a command whose own CLI \
2635                                 never confirmed an exit status"
2636                                .to_owned();
2637                        }
2638                        Err(e) => last_err = e.to_string(),
2639                    }
2640                }
2641                AgentOutcome::Quota(o) => {
2642                    cumulative_wait_ms += o.duration_ms;
2643                    self.state.quota.push(QuotaLoss {
2644                        seat: seat.key.clone(),
2645                        node: "fix".to_owned(),
2646                        at: Timestamp::now(),
2647                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2648                    });
2649                    self.state.event(
2650                        "fix",
2651                        format!(
2652                            "round {round}: continuation rate limited (quota); not retrying now"
2653                        ),
2654                    );
2655                    return (
2656                        seat,
2657                        None,
2658                        Some("rate limited (quota) while recovering the fix report".to_owned()),
2659                        ContinuationRecord {
2660                            attempts,
2661                            cumulative_wait_ms,
2662                            outcome: ContinuationOutcome::QuotaLost,
2663                        },
2664                    );
2665                }
2666                AgentOutcome::Dropped(o) => {
2667                    cumulative_wait_ms += o.duration_ms;
2668                    let why = o
2669                        .dropped
2670                        .as_ref()
2671                        .map(|d| d.why.as_str())
2672                        .unwrap_or("the CLI ended the stream without delivering its answer");
2673                    last_err = format!("the CLI dropped the stream ({why})");
2674                }
2675                AgentOutcome::Failed(e) => last_err = e,
2676                // The continuation was never started; the report stays lost
2677                // and the caller records that, without any handover.
2678                AgentOutcome::ResourceBlocked(e) => {
2679                    return (
2680                        seat,
2681                        None,
2682                        Some(format!("the continuation was not started: {e}")),
2683                        ContinuationRecord {
2684                            attempts: attempts.saturating_sub(1),
2685                            cumulative_wait_ms,
2686                            outcome: ContinuationOutcome::NoSession,
2687                        },
2688                    );
2689                }
2690            }
2691        }
2692    }
2693
2694    fn after_implement(&mut self) -> Result<()> {
2695        // Scan every candidate patch once the set is complete.
2696        if self.state.leaks.is_empty() {
2697            let cfg = self.state.config.blind.clone();
2698            let mut leaks = Vec::new();
2699            for c in &self.state.candidates {
2700                let Some(patch) =
2701                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2702                else {
2703                    continue;
2704                };
2705                leaks.extend(blind::scan(
2706                    &format!("candidate {} patch", c.label),
2707                    &patch,
2708                    &cfg.vendor_tokens,
2709                ));
2710            }
2711            if !leaks.is_empty() {
2712                let summary = leaks
2713                    .iter()
2714                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
2715                    .collect::<Vec<_>>()
2716                    .join(", ");
2717                match cfg.on_leak {
2718                    LeakPolicy::Fail => {
2719                        self.state.status = RunStatus::Failed;
2720                        self.state
2721                            .event("blind", format!("vendor text in a patch: {summary}"));
2722                        self.state.leaks = leaks;
2723                        self.state.save()?;
2724                        self.settle_questions();
2725                        bail!(
2726                            "blind.on_leak = \"fail\" and vendor text reached a \
2727                             judged patch: {summary}"
2728                        );
2729                    }
2730                    LeakPolicy::Redact => self.state.event(
2731                        "blind",
2732                        format!("redacting vendor text for judging: {summary}"),
2733                    ),
2734                    LeakPolicy::Warn => self.state.event(
2735                        "blind",
2736                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
2737                    ),
2738                }
2739                self.state.leaks = leaks;
2740            }
2741        }
2742
2743        if self.state.viable().is_empty() {
2744            if self.state.all_candidates_verified_noop() {
2745                // Every candidate agreed, with evidence the adoption guard
2746                // accepted, that nothing belongs in this worktree. That is
2747                // not the same fact as a candidate that simply failed to
2748                // write anything, and settling it as an ordinary `Failed`
2749                // (see `SCHEMA`'s doc for schema 10) is what let two of
2750                // task 391f's attempts burn a retry each re-discovering the
2751                // same already-landed fix. Terminal either way, so `judge`
2752                // must never run over an empty candidate set — unlike the
2753                // `Failed` branch below this returns `Ok`, not an error:
2754                // nothing here failed.
2755                self.state.status = RunStatus::VerifiedNoop;
2756                self.state.save()?;
2757                self.settle_questions();
2758                return Ok(());
2759            }
2760            self.state.status = RunStatus::Failed;
2761            self.state.save()?;
2762            self.settle_questions();
2763            bail!("no candidate produced a change; nothing to judge");
2764        }
2765        self.state.status = RunStatus::Judging;
2766        self.state.save()?;
2767        Ok(())
2768    }
2769
2770    // --------------------------------------------------------------- judge
2771
2772    async fn judge(&mut self) -> Result<()> {
2773        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2774        // agent files with `magi task add` name the run that paid for it. The
2775        // prompt overlay is cloned alongside it because the waves borrow it
2776        // while `self` is mutably borrowed by the node's own bookkeeping.
2777        let run_id = self.state.id.clone();
2778        let prompts = self.state.config.prompts.clone();
2779        if !self.state.judgements.is_empty() || self.state.judge_skipped {
2780            return Ok(());
2781        }
2782        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2783        if viable.len() == 1 {
2784            // Recorded so this is a one-time event: `judgements` stays empty
2785            // either way, which without this flag is indistinguishable from
2786            // "not yet judged" on the next reentry — and status is left
2787            // untouched, so a later node's conclusion (e.g. `Blocked` after
2788            // the review budget ran out) survives a resume instead of being
2789            // clobbered back to `Judging` by this node running again.
2790            self.state.judge_skipped = true;
2791            self.state.event(
2792                "judge",
2793                format!(
2794                    "only candidate {} produced a change; judging skipped",
2795                    viable[0].label
2796                ),
2797            );
2798            self.state.save()?;
2799            return Ok(());
2800        }
2801        self.state.status = RunStatus::Judging;
2802
2803        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2804        let language = self.state.config.graph.language.clone();
2805        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2806        let sessions = self.state.config.graph.sessions;
2807        let artifacts = agent::artifacts_dir(&self.state.dir());
2808        let root = self.state.worktree_root();
2809        let base_short = short(&self.state.base_commit);
2810
2811        let mut jobs = Vec::new();
2812        let mut orders = Vec::new();
2813        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2814            let order = blind::presentation_order(viable.len(), j, self.state.seed);
2815            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2816            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
2817            let seat_key = format!("judge-{}", j + 1);
2818            let seat = self.seat(&seat_key, &spec.id);
2819            jobs.push(SeatJob {
2820                prompt: prompt::judge(
2821                    &self.state.instruction,
2822                    &views,
2823                    self.roles.judges.len(),
2824                    &base_short,
2825                    &language,
2826                ),
2827                spec,
2828                seat,
2829                cwd: root.join(format!("judge-{}", j + 1)),
2830                timeout,
2831                allow_write: false,
2832                sessions,
2833                artifacts: artifacts.clone(),
2834                stem: format!("judge-{}", j + 1),
2835                handover: None,
2836            });
2837        }
2838
2839        self.state.event(
2840            "judge",
2841            format!(
2842                "{} judges ranking {} candidates blind",
2843                jobs.len(),
2844                viable.len()
2845            ),
2846        );
2847        let labels_for_check = labels.clone();
2848        let mut quota_losses = Vec::new();
2849        let cache = self.state.config.cache_dir();
2850        let ctx = WaveCtx {
2851            carry_seats: false,
2852            pauses: Some((&self.pause, &self.interrupt)),
2853            run: &run_id,
2854            node: "judge",
2855            prompts: &prompts,
2856            cache: cache.as_deref(),
2857            round: None,
2858        };
2859        let results = ask_json_wave::<Ranking>(
2860            jobs,
2861            Arc::clone(&self.sem),
2862            self.state.config.graph.retries,
2863            &self.roles.judge_roster,
2864            &ctx,
2865            &mut quota_losses,
2866            &mut self.state,
2867            &move |r: &Ranking| r.validate(&labels_for_check),
2868        )
2869        .await;
2870        if self.state.resource_block.is_some() {
2871            return self.stop_for_resource();
2872        }
2873        self.state.quota.extend(quota_losses);
2874
2875        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
2876            let agent_id = seat.agent.clone();
2877            self.state.seats.insert(seat.key.clone(), seat);
2878            let mut record = Judgement {
2879                judge: j + 1,
2880                seat: format!("judge-{}", j + 1),
2881                agent: agent_id,
2882                ranking: Vec::new(),
2883                reasons: BTreeMap::new(),
2884                confidence: None,
2885                strengths: BTreeMap::new(),
2886                order: orders[j].clone(),
2887                failed: None,
2888                duration_ms: 0,
2889            };
2890            match res {
2891                Ok((ranking, out)) => {
2892                    record.ranking = ranking.normalized();
2893                    record.reasons = ranking.reasons;
2894                    record.strengths = ranking.strengths;
2895                    record.confidence = ranking.confidence;
2896                    record.duration_ms = out.duration_ms;
2897                    self.state.event(
2898                        "judge",
2899                        format!(
2900                            "judge {} ranked {}",
2901                            j + 1,
2902                            record.ranking.iter().collect::<String>()
2903                        ),
2904                    );
2905                }
2906                Err(e) => {
2907                    record.failed = Some(e.to_string());
2908                    self.state
2909                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
2910                }
2911            }
2912            self.state.judgements.push(record);
2913            self.state.save()?;
2914        }
2915        Ok(())
2916    }
2917
2918    // ---------------------------------------------------------- deliberate
2919
2920    async fn deliberate(&mut self) -> Result<()> {
2921        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2922        // agent files with `magi task add` name the run that paid for it. The
2923        // prompt overlay is cloned alongside it because the waves borrow it
2924        // while `self` is mutably borrowed by the node's own bookkeeping.
2925        let run_id = self.state.id.clone();
2926        let prompts = self.state.config.prompts.clone();
2927        if !self.state.deliberation.is_empty() {
2928            return Ok(());
2929        }
2930        let tops: Vec<char> = self
2931            .state
2932            .judgements
2933            .iter()
2934            .filter_map(|j| j.ranking.first().copied())
2935            .collect();
2936        let rounds = self.state.config.graph.deliberate_rounds;
2937        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
2938            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
2939                self.state.event(
2940                    "deliberate",
2941                    format!("judges agreed on {} outright; no deliberation", tops[0]),
2942                );
2943            }
2944            self.state.status = RunStatus::Voting;
2945            self.state.save()?;
2946            return Ok(());
2947        }
2948
2949        self.state.status = RunStatus::Deliberating;
2950        self.state.event(
2951            "deliberate",
2952            format!(
2953                "split: first choices were {} — opening {rounds} round(s)",
2954                tops.iter().collect::<String>()
2955            ),
2956        );
2957
2958        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2959        let language = self.state.config.graph.language.clone();
2960        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2961        let sessions = self.state.config.graph.sessions;
2962        let artifacts = agent::artifacts_dir(&self.state.dir());
2963        let root = self.state.worktree_root();
2964        let base_short = short(&self.state.base_commit);
2965
2966        // Judges argue in sequence so that a turn can answer the one before it;
2967        // that is the difference between deliberation and three parallel
2968        // monologues.
2969        for round in 1..=rounds {
2970            let mut turns: Vec<DeliberationTurn> = Vec::new();
2971            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2972                if self.state.judgements[j].failed.is_some() {
2973                    continue;
2974                }
2975                let seat_key = format!("judge-{}", j + 1);
2976                let spec = self.occupant(&seat_key, spec);
2977                let mut seat = self.seat(&seat_key, &spec.id);
2978                let transcript = self.transcript(&turns, j);
2979                let build = |context: Option<&str>| {
2980                    prompt::deliberate(
2981                        &self.state.instruction,
2982                        context,
2983                        &transcript,
2984                        round,
2985                        rounds,
2986                        &language,
2987                    )
2988                };
2989                let block = self.candidate_block(&viable, &base_short);
2990                let full = build(Some(&block));
2991                let text = if has_context(&spec, &seat, sessions) {
2992                    build(None)
2993                } else {
2994                    full.clone()
2995                };
2996                let job = SeatJob {
2997                    spec,
2998                    seat: seat.clone(),
2999                    prompt: text,
3000                    cwd: root.join(format!("judge-{}", j + 1)),
3001                    timeout,
3002                    allow_write: false,
3003                    sessions,
3004                    artifacts: artifacts.clone(),
3005                    stem: format!("delib-{round}-judge-{}", j + 1),
3006                    handover: Some(full),
3007                };
3008                let cache = self.state.config.cache_dir();
3009                let ctx = WaveCtx {
3010                    carry_seats: false,
3011                    pauses: Some((&self.pause, &self.interrupt)),
3012                    run: &run_id,
3013                    node: "deliberate",
3014                    prompts: &prompts,
3015                    cache: cache.as_deref(),
3016                    round: None,
3017                };
3018                // A turn is never nudged (`retries` 0): a failed seat is
3019                // handed to the next roster agent, which gets the full
3020                // context. An empty answer is a turn, not a failure.
3021                let mut losses = Vec::new();
3022                let mut results = ask_wave_with::<String>(
3023                    vec![job],
3024                    Arc::clone(&self.sem),
3025                    0,
3026                    &self.roles.judge_roster,
3027                    &ctx,
3028                    &mut losses,
3029                    &mut self.state,
3030                    &|text: &str| {
3031                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3032                    },
3033                )
3034                .await;
3035                self.state.quota.extend(losses);
3036                let (updated, res, _) = results.pop().expect("one job in, one result out");
3037                seat = updated;
3038                let agent_id = seat.agent.clone();
3039                self.state.seats.insert(seat.key.clone(), seat);
3040                let body = match res {
3041                    Ok((body, _)) => body,
3042                    // Skip the seat; a CLI's raw error JSON is never read as
3043                    // this judge's position.
3044                    Err(e) => {
3045                        self.state
3046                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3047                        continue;
3048                    }
3049                };
3050                let tentative = verdict::extract_json::<Position>(&body)
3051                    .ok()
3052                    .and_then(|p| p.tentative)
3053                    .and_then(|s| s.trim().chars().next())
3054                    .map(|c| c.to_ascii_uppercase());
3055                self.state.event(
3056                    "deliberate",
3057                    format!(
3058                        "round {round}: judge {} now favours {}",
3059                        j + 1,
3060                        tentative.map_or("—".to_owned(), |c| c.to_string())
3061                    ),
3062                );
3063                turns.push(DeliberationTurn {
3064                    judge: j + 1,
3065                    agent: agent_id,
3066                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3067                    tentative,
3068                });
3069            }
3070            self.state
3071                .deliberation
3072                .push(DeliberationRound { round, turns });
3073            self.state.save()?;
3074        }
3075
3076        self.state.status = RunStatus::Voting;
3077        self.state.save()?;
3078        Ok(())
3079    }
3080
3081    // ---------------------------------------------------------------- vote
3082
3083    async fn vote(&mut self) -> Result<()> {
3084        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3085        // agent files with `magi task add` name the run that paid for it. The
3086        // prompt overlay is cloned alongside it because the waves borrow it
3087        // while `self` is mutably borrowed by the node's own bookkeeping.
3088        let run_id = self.state.id.clone();
3089        let prompts = self.state.config.prompts.clone();
3090        if !self.state.votes.is_empty() {
3091            return Ok(());
3092        }
3093        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3094        if viable.len() == 1 {
3095            return Ok(());
3096        }
3097        self.state.status = RunStatus::Voting;
3098
3099        let language = self.state.config.graph.language.clone();
3100        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3101        let sessions = self.state.config.graph.sessions;
3102        let artifacts = agent::artifacts_dir(&self.state.dir());
3103        let root = self.state.worktree_root();
3104        let base_short = short(&self.state.base_commit);
3105        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3106
3107        let mut jobs = Vec::new();
3108        let mut seats_at = Vec::new();
3109        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3110            if self
3111                .state
3112                .judgements
3113                .get(j)
3114                .is_some_and(|r| r.failed.is_some())
3115            {
3116                continue;
3117            }
3118            let seat_key = format!("judge-{}", j + 1);
3119            let spec = self.occupant(&seat_key, spec);
3120            let seat = self.seat(&seat_key, &spec.id);
3121            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3122            let text = if has_context(&spec, &seat, sessions) {
3123                prompt::final_vote(&viable, &language)
3124            } else {
3125                full.clone()
3126            };
3127            jobs.push(SeatJob {
3128                spec,
3129                seat,
3130                prompt: text,
3131                cwd: root.join(format!("judge-{}", j + 1)),
3132                timeout,
3133                allow_write: false,
3134                sessions,
3135                artifacts: artifacts.clone(),
3136                stem: format!("vote-judge-{}", j + 1),
3137                handover: Some(full),
3138            });
3139            seats_at.push(j);
3140        }
3141
3142        self.state.event(
3143            "vote",
3144            format!(
3145                "collecting {} final votes one by one, privately",
3146                jobs.len()
3147            ),
3148        );
3149        let allowed = viable.clone();
3150        let mut quota_losses = Vec::new();
3151        let cache = self.state.config.cache_dir();
3152        let ctx = WaveCtx {
3153            carry_seats: false,
3154            pauses: Some((&self.pause, &self.interrupt)),
3155            run: &run_id,
3156            node: "vote",
3157            prompts: &prompts,
3158            cache: cache.as_deref(),
3159            round: None,
3160        };
3161        let results = ask_json_wave::<FinalVote>(
3162            jobs,
3163            Arc::clone(&self.sem),
3164            self.state.config.graph.retries,
3165            &self.roles.judge_roster,
3166            &ctx,
3167            &mut quota_losses,
3168            &mut self.state,
3169            &move |v: &FinalVote| match v.label() {
3170                Some(c) if allowed.contains(&c) => Ok(()),
3171                other => bail!("vote {other:?} is not one of {allowed:?}"),
3172            },
3173        )
3174        .await;
3175        if self.state.resource_block.is_some() {
3176            return self.stop_for_resource();
3177        }
3178        self.state.quota.extend(quota_losses);
3179
3180        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3181            let agent_id = seat.agent.clone();
3182            self.state.seats.insert(seat.key.clone(), seat);
3183            let initial = self
3184                .state
3185                .judgements
3186                .get(j)
3187                .and_then(|r| r.ranking.first().copied());
3188            let mut record = VoteRecord {
3189                judge: j + 1,
3190                agent: agent_id,
3191                vote: None,
3192                reason: String::new(),
3193                changed: false,
3194            };
3195            match res {
3196                Ok((v, _)) => {
3197                    record.vote = v.label();
3198                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3199                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3200                    self.state.event(
3201                        "vote",
3202                        format!(
3203                            "judge {} voted {}{}",
3204                            j + 1,
3205                            record.vote.unwrap_or('?'),
3206                            if record.changed { " (changed)" } else { "" }
3207                        ),
3208                    );
3209                }
3210                Err(e) => {
3211                    self.state
3212                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3213                }
3214            }
3215            self.state.votes.push(record);
3216            self.state.save()?;
3217        }
3218        Ok(())
3219    }
3220
3221    // --------------------------------------------------------------- tally
3222
3223    fn tally(&mut self) -> Result<()> {
3224        if self.state.tally.is_some() {
3225            return Ok(());
3226        }
3227        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3228        let tops: Vec<char> = self
3229            .state
3230            .judgements
3231            .iter()
3232            .filter_map(|j| j.ranking.first().copied())
3233            .collect();
3234        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3235
3236        // A judge whose private vote failed still counted once, in the initial
3237        // ranking; using it beats discarding a whole seat.
3238        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3239        let mut cast: Vec<char> = Vec::new();
3240        for (i, j) in self.state.judgements.iter().enumerate() {
3241            let vote = self
3242                .state
3243                .votes
3244                .iter()
3245                .find(|v| v.judge == i + 1)
3246                .and_then(|v| v.vote)
3247                .or_else(|| j.ranking.first().copied());
3248            if let Some(v) = vote {
3249                *first_choice.entry(v).or_insert(0) += 1;
3250                cast.push(v);
3251            }
3252        }
3253
3254        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3255        for j in &self.state.judgements {
3256            let n = j.ranking.len();
3257            for (pos, label) in j.ranking.iter().enumerate() {
3258                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3259            }
3260        }
3261
3262        let best = first_choice.values().copied().max().unwrap_or(0);
3263        let mut leaders: Vec<char> = first_choice
3264            .iter()
3265            .filter(|(_, v)| **v == best)
3266            .map(|(k, _)| *k)
3267            .collect();
3268        let mut tie_break = None;
3269        if leaders.len() > 1 {
3270            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3271            let borda_leaders: Vec<char> = leaders
3272                .iter()
3273                .copied()
3274                .filter(|l| borda[l] == top_borda)
3275                .collect();
3276            tie_break = Some(if borda_leaders.len() == 1 {
3277                format!(
3278                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3279                    leaders.len()
3280                )
3281            } else {
3282                format!(
3283                    "{} way tie on both first-choice votes and Borda points, broken by label order",
3284                    leaders.len()
3285                )
3286            });
3287            leaders = borda_leaders;
3288            leaders.sort_unstable();
3289        }
3290        let winner = *leaders
3291            .first()
3292            .or(viable.first())
3293            .context("no candidate to declare a winner from")?;
3294
3295        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3296        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3297        let deliberated = !self.state.deliberation.is_empty();
3298
3299        // Whose verdict is this? A rate-limited seat is absent even if it
3300        // ranked before the limit hit, so presence is measured against the
3301        // recorded losses, not just "did a ranking ever appear".
3302        let quota_seats: std::collections::BTreeSet<&str> =
3303            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3304        let mut present = 0usize;
3305        for (i, j) in self.state.judgements.iter().enumerate() {
3306            if quota_seats.contains(j.seat.as_str()) {
3307                continue;
3308            }
3309            let ranked = !j.ranking.is_empty() && j.failed.is_none();
3310            let voted = self
3311                .state
3312                .votes
3313                .iter()
3314                .any(|v| v.judge == i + 1 && v.vote.is_some());
3315            if ranked || voted {
3316                present += 1;
3317            }
3318        }
3319        // Strict majority of the configured panel. A bare majority is real
3320        // signal we can act on, while a minority verdict must never stand in
3321        // for a healthy one. A one-candidate run needs no panel at all, and
3322        // `judges` stays `0` rather than the roster size a panel that never
3323        // sat would otherwise be credited with.
3324        let needs_quorum = viable.len() > 1;
3325        let judges_total = if needs_quorum {
3326            self.roles.judges.len()
3327        } else {
3328            0
3329        };
3330        let quorum = if needs_quorum {
3331            judges_total / 2 + 1
3332        } else {
3333            0
3334        };
3335        let met_quorum = !needs_quorum || present >= quorum;
3336        let uncontested = (!needs_quorum).then(|| {
3337            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3338        });
3339
3340        self.state.event(
3341            "tally",
3342            match &uncontested {
3343                Some(reason) => format!("winner {winner} — {reason}"),
3344                None => format!(
3345                    "winner {winner} — votes {} | initial {} | {} changed | \
3346                     {present}/{judges_total} judges{}",
3347                    first_choice
3348                        .iter()
3349                        .map(|(k, v)| format!("{k}:{v}"))
3350                        .collect::<Vec<_>>()
3351                        .join(" "),
3352                    if unanimous_initial {
3353                        "unanimous"
3354                    } else {
3355                        "split"
3356                    },
3357                    changed_votes,
3358                    if met_quorum {
3359                        String::new()
3360                    } else {
3361                        format!(" — below quorum ({quorum} required)")
3362                    },
3363                ),
3364            },
3365        );
3366        if !met_quorum {
3367            self.state.event(
3368                "stall",
3369                format!(
3370                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3371                     the run stops here, resumable"
3372                ),
3373            );
3374        }
3375        self.state.tally = Some(Tally {
3376            first_choice,
3377            borda,
3378            winner,
3379            rankings: tops.len(),
3380            unanimous_initial,
3381            deliberated,
3382            changed_votes,
3383            unanimous_final,
3384            tie_break,
3385            judges: judges_total,
3386            present,
3387            quorum,
3388            met_quorum,
3389            uncontested,
3390        });
3391        self.state.status = if met_quorum {
3392            RunStatus::Reviewing
3393        } else {
3394            RunStatus::Stalled
3395        };
3396        self.state.save()?;
3397        Ok(())
3398    }
3399
3400    // ------------------------------------------------------------- recover
3401
3402    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
3403    /// resumed toward completion once the transient cause clears.
3404    ///
3405    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
3406    /// it toward the quorum, which is exactly the set of seats whose absence
3407    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
3408    /// not depend on which node happened to hit the limit), or an ordinary
3409    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
3410    /// seat is never disturbed.
3411    ///
3412    /// A seat that now answers with a usable ranking is "recovered": its
3413    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
3414    /// `tally` counts it present again), and its vote re-collected. A seat that
3415    /// still fails keeps its loss and stays absent.
3416    ///
3417    /// Returns `true` when the re-tally restores the quorum (the run may proceed
3418    /// to review/gate/merge), `false` when it is still below quorum (the run
3419    /// stays `Stalled`, still resumable for a later retry).
3420    #[allow(clippy::too_many_lines)]
3421    async fn recover_stall(&mut self) -> Result<bool> {
3422        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3423        // agent files with `magi task add` name the run that paid for it. The
3424        // prompt overlay is cloned alongside it because the waves borrow it
3425        // while `self` is mutably borrowed by the node's own bookkeeping.
3426        let run_id = self.state.id.clone();
3427        let prompts = self.state.config.prompts.clone();
3428        // Absent seats = quota-lost at any node, or failed outright. Mirroring
3429        // `tally`'s presence test (rather than the old quota-judge/vote filter)
3430        // is what keeps a non-quota collapse — or a quota loss recorded at the
3431        // deliberate node — from being a permanent dead-end on `--resume`.
3432        let quota_seats: BTreeSet<&str> =
3433            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3434        let absent: Vec<String> = self
3435            .state
3436            .judgements
3437            .iter()
3438            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3439            .map(|j| j.seat.clone())
3440            .collect();
3441        if absent.is_empty() {
3442            return Ok(false);
3443        }
3444        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3445        if viable.len() <= 1 {
3446            return Ok(false);
3447        }
3448        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3449        let language = self.state.config.graph.language.clone();
3450        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3451        let sessions = self.state.config.graph.sessions;
3452        let artifacts = agent::artifacts_dir(&self.state.dir());
3453        let root = self.state.worktree_root();
3454        let base_short = short(&self.state.base_commit);
3455        let candidates: Vec<Candidate> = viable.clone();
3456
3457        // Map each absent seat key to its 0-based position in `roles.judges`.
3458        let mut positions: Vec<usize> = absent
3459            .iter()
3460            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3461            .collect();
3462        if positions.is_empty() {
3463            return Ok(false);
3464        }
3465        positions.sort_unstable();
3466        positions.dedup();
3467
3468        // Re-rank the lost seats, one blind prompt each.
3469        let mut judge_jobs = Vec::new();
3470        for &j in &positions {
3471            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3472            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3473            let seat_key = format!("judge-{}", j + 1);
3474            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3475            let seat = self.seat(&seat_key, &spec.id);
3476            judge_jobs.push(SeatJob {
3477                spec,
3478                seat,
3479                prompt: prompt::judge(
3480                    &self.state.instruction,
3481                    &views,
3482                    self.roles.judges.len(),
3483                    &base_short,
3484                    &language,
3485                ),
3486                cwd: root.join(seat_key),
3487                timeout,
3488                allow_write: false,
3489                sessions,
3490                artifacts: artifacts.clone(),
3491                stem: format!("judge-{}-recover", j + 1),
3492                handover: None,
3493            });
3494        }
3495
3496        let labels_for_check = labels.clone();
3497        let mut judge_losses = Vec::new();
3498        let retries = self.state.config.graph.retries;
3499        let cache = self.state.config.cache_dir();
3500        let ctx = WaveCtx {
3501            carry_seats: false,
3502            pauses: Some((&self.pause, &self.interrupt)),
3503            run: &run_id,
3504            node: "judge",
3505            prompts: &prompts,
3506            cache: cache.as_deref(),
3507            round: None,
3508        };
3509        let results = ask_json_wave::<Ranking>(
3510            judge_jobs,
3511            Arc::clone(&self.sem),
3512            retries,
3513            &self.roles.judge_roster,
3514            &ctx,
3515            &mut judge_losses,
3516            &mut self.state,
3517            &move |r: &Ranking| r.validate(&labels_for_check),
3518        )
3519        .await;
3520        if self.state.resource_block.is_some() {
3521            self.stop_for_resource()?;
3522            return Ok(false);
3523        }
3524
3525        // Refresh the judgement of every seat that ranked again.
3526        let mut recovered: BTreeSet<usize> = BTreeSet::new();
3527        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3528            let agent_id = seat.agent.clone();
3529            self.state.seats.insert(seat.key.clone(), seat);
3530            let record = &mut self.state.judgements[j];
3531            match res {
3532                Ok((ranking, out)) => {
3533                    record.agent = agent_id;
3534                    record.ranking = ranking.normalized();
3535                    record.reasons = ranking.reasons;
3536                    record.strengths = ranking.strengths;
3537                    record.confidence = ranking.confidence;
3538                    record.failed = None;
3539                    record.duration_ms = out.duration_ms;
3540                    recovered.insert(j);
3541                    self.state.event(
3542                        "recover",
3543                        format!("judge {} ranked again after the limit", j + 1),
3544                    );
3545                }
3546                Err(e) => {
3547                    self.state
3548                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3549                }
3550            }
3551        }
3552
3553        // Re-ask the votes of the seats that recovered a ranking.
3554        let mut vote_jobs = Vec::new();
3555        let mut vote_pos: Vec<usize> = Vec::new();
3556        for &j in &recovered {
3557            let seat_key = format!("judge-{}", j + 1);
3558            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3559            let seat = self.seat(&seat_key, &spec.id);
3560            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
3561            let text = if has_context(&spec, &seat, sessions) {
3562                prompt::final_vote(&labels, &language)
3563            } else {
3564                full.clone()
3565            };
3566            vote_jobs.push(SeatJob {
3567                spec,
3568                seat,
3569                prompt: text,
3570                cwd: root.join(seat_key),
3571                timeout,
3572                allow_write: false,
3573                sessions,
3574                artifacts: artifacts.clone(),
3575                stem: format!("vote-judge-{}-recover", j + 1),
3576                handover: Some(full),
3577            });
3578            vote_pos.push(j);
3579        }
3580        let allowed = labels.clone();
3581        let mut vote_losses = Vec::new();
3582        let vote_retries = self.state.config.graph.retries;
3583        let vote_cache = self.state.config.cache_dir();
3584        let ctx = WaveCtx {
3585            carry_seats: false,
3586            pauses: Some((&self.pause, &self.interrupt)),
3587            run: &run_id,
3588            node: "vote",
3589            prompts: &prompts,
3590            cache: vote_cache.as_deref(),
3591            round: None,
3592        };
3593        let votes = ask_json_wave::<FinalVote>(
3594            vote_jobs,
3595            Arc::clone(&self.sem),
3596            vote_retries,
3597            &self.roles.judge_roster,
3598            &ctx,
3599            &mut vote_losses,
3600            &mut self.state,
3601            &move |v: &FinalVote| match v.label() {
3602                Some(c) if allowed.contains(&c) => Ok(()),
3603                other => bail!("vote {other:?} is not one of {allowed:?}"),
3604            },
3605        )
3606        .await;
3607        if self.state.resource_block.is_some() {
3608            self.stop_for_resource()?;
3609            return Ok(false);
3610        }
3611        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
3612            let agent_id = seat.agent.clone();
3613            self.state.seats.insert(seat.key.clone(), seat);
3614            match res {
3615                Ok((v, _)) => {
3616                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
3617                        rec.vote = v.label();
3618                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3619                    } else {
3620                        self.state.votes.push(VoteRecord {
3621                            judge: j + 1,
3622                            agent: agent_id,
3623                            vote: v.label(),
3624                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
3625                            changed: false,
3626                        });
3627                    }
3628                    self.state.event(
3629                        "recover",
3630                        format!("judge {} voted again after the limit", j + 1),
3631                    );
3632                }
3633                Err(e) => {
3634                    self.state
3635                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
3636                }
3637            }
3638        }
3639
3640        // A seat that ranked again is present even if its re-vote failed —
3641        // `tally` falls back to the initial ranking's first choice — so clear
3642        // its quota loss. Seats that still fail keep theirs and stay absent.
3643        let recovered_keys: BTreeSet<String> = recovered
3644            .iter()
3645            .map(|&j| format!("judge-{}", j + 1))
3646            .collect();
3647        self.state
3648            .quota
3649            .retain(|q| !recovered_keys.contains(&q.seat));
3650        // A seat that hit the limit again is a fresh loss, not the old one:
3651        // replace the stale entry so the history stays one-per-seat and the
3652        // daemon can tell this attempt's loss from a previous session's.
3653        for loss in judge_losses.into_iter().chain(vote_losses) {
3654            if recovered_keys.contains(&loss.seat) {
3655                continue;
3656            }
3657            self.state.quota.retain(|q| q.seat != loss.seat);
3658            self.state.quota.push(loss);
3659        }
3660
3661        // Recompute the verdict from the refreshed panel.
3662        self.state.tally = None;
3663        self.tally()?;
3664        Ok(self
3665            .state
3666            .tally
3667            .as_ref()
3668            .map(|t| t.met_quorum)
3669            .unwrap_or(false))
3670    }
3671
3672    // ---------------------------------------------------------------- graft
3673
3674    /// Port the strengths the judges attributed to the losing candidates onto
3675    /// the winner, once, between the tally and the review loop.
3676    ///
3677    /// Best-effort by construction: it returns nothing and fails nothing. A
3678    /// skip, a fixer that failed or hit quota, a round that changed nothing
3679    /// and a result over the size limit ([`graft::limit`]) all leave the
3680    /// winner exactly at the commit it had before (`GraftRecord::pre`) and the
3681    /// run carries on as if the phase were off. See [`graft`] for the
3682    /// decisions and `AGENTS.md` for the invariants.
3683    ///
3684    /// The one exception is a rollback that could not be confirmed: the
3685    /// rejected commit may still be the winner's tip, so the run is blocked
3686    /// (resumable; the record stays `Running`, which makes a resume retry the
3687    /// restore) instead of reviewing, gating and merging it.
3688    async fn graft(&mut self) -> Result<()> {
3689        let Err(e) = self.graft_inner().await else {
3690            return Ok(());
3691        };
3692        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
3693            self.state.status = RunStatus::Blocked;
3694            self.state.event("graft", format!("blocked: {e:#}"));
3695            self.state.save()?;
3696            return Err(e);
3697        }
3698        // An error after the `Running` record was saved may have left the
3699        // fixer's work in the tree: the record must not stay unsettled
3700        // (a later resume would reset past the review fixes), so confirm the
3701        // restore now, or block.
3702        if let Some(rec) = self.state.graft.clone()
3703            && !rec.outcome.settled()
3704        {
3705            if let Some(winner) = self.state.winner().cloned() {
3706                if let Err(re) = Self::graft_restore(&winner.worktree, &rec.pre).await {
3707                    self.state.status = RunStatus::Blocked;
3708                    self.state.event("graft", format!("blocked: {re:#}"));
3709                    self.state.save()?;
3710                    return Err(re);
3711                }
3712            }
3713            if let Some(g) = self.state.graft.as_mut() {
3714                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
3715            }
3716        }
3717        self.state.event("graft", format!("skipped: {e:#}"));
3718        if let Err(e) = self.state.save() {
3719            tracing::warn!("save after graft: {e:#}");
3720        }
3721        Ok(())
3722    }
3723
3724    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
3725    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
3726    async fn graft_restore(wt: &Path, pre: &str) -> Result<()> {
3727        let restored = git::restore_to(wt, pre).await;
3728        let tip = git::rev_parse(wt, "HEAD").await;
3729        match (restored, tip) {
3730            (Ok(()), Ok(tip)) if tip == pre => Ok(()),
3731            (Err(e), _) => Err(GraftRestoreFailed(format!("{e:#}")).into()),
3732            (Ok(()), Ok(tip)) => {
3733                Err(GraftRestoreFailed(format!("the winner is at {tip}, not {pre}")).into())
3734            }
3735            (Ok(()), Err(e)) => Err(GraftRestoreFailed(format!("{e:#}")).into()),
3736        }
3737    }
3738
3739    /// A skip is a run event, recorded once however often the node is re-entered.
3740    fn graft_skipped(&mut self, why: &str) {
3741        let message = format!("skipped: {why}");
3742        if !self
3743            .state
3744            .events
3745            .iter()
3746            .any(|e| e.node == "graft" && e.message == message)
3747        {
3748            self.state.event("graft", message);
3749        }
3750    }
3751
3752    async fn graft_inner(&mut self) -> Result<()> {
3753        // A round that was started and never finished (crash, kill): the
3754        // fixer may have left anything in the tree, so put the winner back.
3755        if let Some(rec) = self.state.graft.clone()
3756            && !rec.outcome.settled()
3757        {
3758            // Reviews exist only if the graft ended without its record being
3759            // settled (a swallowed error): the tree has moved on since, and
3760            // a reset to `pre` would discard the review fixes.
3761            if self.state.reviews.is_empty()
3762                && let Some(winner) = self.state.winner().cloned()
3763            {
3764                Self::graft_restore(&winner.worktree, &rec.pre).await?;
3765            }
3766            if let Some(g) = self.state.graft.as_mut() {
3767                g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
3768            }
3769            self.state.event(
3770                "graft",
3771                "rolled back: the previous round was interrupted; the winner is untouched",
3772            );
3773            self.state.save()?;
3774            return Ok(());
3775        }
3776        let plan = match graft::plan(&self.state) {
3777            Ok(plan) => plan,
3778            Err(skip) => {
3779                if skip != graft::Skip::AlreadyDone {
3780                    self.graft_skipped(skip.reason());
3781                    self.state.save()?;
3782                }
3783                return Ok(());
3784            }
3785        };
3786        let Some(winner) = self.state.winner().cloned() else {
3787            return Ok(());
3788        };
3789        let wt = winner.worktree.clone();
3790        if !git::is_clean(&wt).await? {
3791            self.graft_skipped("the winner's worktree has uncommitted changes");
3792            self.state.save()?;
3793            return Ok(());
3794        }
3795        let pre = git::rev_parse(&wt, "HEAD").await?;
3796        let Some(winner_lines) = git::changed_lines(&wt, &self.state.base_commit, &pre).await
3797        else {
3798            self.graft_skipped("the winner's diff could not be measured");
3799            self.state.save()?;
3800            return Ok(());
3801        };
3802        let limit = graft::limit(winner_lines);
3803        // Saved before the fixer is called: a stop from here on is repaired
3804        // by the branch above on the next entry.
3805        self.state.graft = Some(GraftRecord {
3806            pre: pre.clone(),
3807            post: None,
3808            lines: None,
3809            limit: Some(limit),
3810            agent: None,
3811            outcome: GraftOutcome::Running,
3812            requested: graft::requested(&plan),
3813            adoption: None,
3814            adoption_error: None,
3815        });
3816        self.state.event(
3817            "graft",
3818            format!(
3819                "porting strengths of candidate(s) {} onto {} (limit {limit} changed lines)",
3820                plan.strengths
3821                    .iter()
3822                    .map(|(l, _, _)| l.to_string())
3823                    .collect::<Vec<_>>()
3824                    .join(", "),
3825                plan.winner
3826            ),
3827        );
3828        self.state.save()?;
3829
3830        let fix_prompt = prompt::graft(
3831            &self.state.instruction,
3832            plan.winner,
3833            &plan.strengths,
3834            limit,
3835            &self.state.config.graph.language,
3836        );
3837        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
3838        let sessions = self.state.config.graph.sessions;
3839        let artifacts = agent::artifacts_dir(&self.state.dir());
3840        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
3841        let (_, seat, out) = self
3842            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
3843                SeatJob {
3844                    prompt: fix_prompt.clone(),
3845                    spec,
3846                    seat,
3847                    cwd: wt.clone(),
3848                    timeout,
3849                    allow_write: true,
3850                    sessions,
3851                    artifacts: artifacts.clone(),
3852                    stem: "graft".to_owned(),
3853                    handover: None,
3854                }
3855            })
3856            .await;
3857        // Never spawned: nothing changed (`HEAD == pre` is confirmed), so the
3858        // record goes back to "not started" - not `RolledBack`, which would
3859        // settle it - and the run stops resumably. The seat, the chain and
3860        // the quota are untouched.
3861        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3862            Self::graft_restore(&wt, &pre).await?;
3863            self.state.graft = None;
3864            self.state.save()?;
3865            return self.stop_for_resource();
3866        }
3867        let mut why: Option<String> = None;
3868        match &out {
3869            AgentOutcome::Ok(o) if agent::output_advances(o) => {
3870                why = Some("the fixer produced nothing usable".to_owned());
3871            }
3872            AgentOutcome::Ok(_) => {}
3873            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
3874            AgentOutcome::Quota(o) => {
3875                self.state.quota.push(QuotaLoss {
3876                    seat: seat.key.clone(),
3877                    node: "graft".to_owned(),
3878                    at: Timestamp::now(),
3879                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3880                });
3881                why = Some("rate limited (quota)".to_owned());
3882            }
3883            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3884                why = Some(format!(
3885                    "the fixer failed: {}",
3886                    e.lines()
3887                        .next()
3888                        .unwrap_or("")
3889                        .chars()
3890                        .take(160)
3891                        .collect::<String>()
3892                ));
3893            }
3894        }
3895        let agent_id = seat.agent.clone();
3896        self.state.seats.insert(seat.key.clone(), seat);
3897
3898        let mut post = None;
3899        let mut lines = None;
3900        if why.is_none() {
3901            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
3902            if let Ok(r) = &rescued {
3903                self.state.note_withheld("graft", &r.withheld);
3904            }
3905            let head = git::rev_parse(&wt, "HEAD").await?;
3906            // What is measured must be what gets reviewed and gated: a failed
3907            // rescue, or anything still uncommitted (withheld lockfiles
3908            // aside, which are never part of the tree), rejects the graft.
3909            let dirty = match &rescued {
3910                Ok(r) => git::status_porcelain(&wt)
3911                    .await
3912                    .map(|st| {
3913                        st.lines().any(|l| {
3914                            !r.withheld
3915                                .iter()
3916                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
3917                        })
3918                    })
3919                    .unwrap_or(true),
3920                Err(_) => true,
3921            };
3922            if dirty {
3923                why = Some("the fixer left changes that could not be committed".to_owned());
3924            } else if head == pre {
3925                why = Some("the fixer changed nothing".to_owned());
3926            } else {
3927                lines = git::changed_lines(&wt, &pre, &head).await;
3928                if graft::within_budget(lines, limit) {
3929                    post = Some(head);
3930                } else {
3931                    why = Some(format!(
3932                        "{} changed lines against a limit of {limit}",
3933                        lines.map_or_else(
3934                            || "an unmeasurable number of".to_owned(),
3935                            |l| l.to_string()
3936                        )
3937                    ));
3938                }
3939            }
3940        }
3941        let (adoption, adoption_error) = match &out {
3942            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
3943            _ => (None, None),
3944        };
3945        let requested = graft::requested(&plan);
3946        let outcome = match why {
3947            None => {
3948                self.state.event(
3949                    "graft",
3950                    format!(
3951                        "applied: {} changed lines (limit {limit})",
3952                        lines.unwrap_or(0)
3953                    ),
3954                );
3955                GraftOutcome::Applied
3956            }
3957            Some(why) => {
3958                // Not settled until the restore is confirmed: the record
3959                // stays `Running`, so a resume retries it.
3960                Self::graft_restore(&wt, &pre).await?;
3961                self.state.event(
3962                    "graft",
3963                    format!("rolled back: {why}; the winner is untouched"),
3964                );
3965                GraftOutcome::RolledBack(why)
3966            }
3967        };
3968        self.state.graft = Some(GraftRecord {
3969            pre,
3970            post,
3971            lines,
3972            limit: Some(limit),
3973            agent: Some(agent_id),
3974            outcome,
3975            requested,
3976            adoption,
3977            adoption_error,
3978        });
3979        self.state.save()?;
3980        Ok(())
3981    }
3982
3983    // ----------------------------------------------------------------- fold
3984
3985    async fn fold_losers(&mut self) -> Result<()> {
3986        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
3987            return Ok(());
3988        };
3989        let repo = self.state.repo.clone();
3990        let mut folded = Vec::new();
3991        for i in 0..self.state.candidates.len() {
3992            let c = &self.state.candidates[i];
3993            if c.label == winner || c.folded {
3994                continue;
3995            }
3996            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
3997            git::worktree_remove(&repo, &wt).await.ok();
3998            git::branch_delete(&repo, &branch).await.ok();
3999            self.state.candidates[i].folded = true;
4000            folded.push(label.to_string());
4001        }
4002        // The judges are finished; their checkouts are pure cost from here.
4003        let root = self.state.worktree_root();
4004        for j in 1..=self.roles.judges.len() {
4005            let wt = root.join(format!("judge-{j}"));
4006            if wt.exists() {
4007                git::worktree_remove(&repo, &wt).await.ok();
4008            }
4009        }
4010        // The design-deliberation stage is finished by the time a tally
4011        // exists — same reasoning as the judges above.
4012        if self.state.config.graph.advise {
4013            for k in 1..=self.state.config.graph.advisors {
4014                let wt = root.join(format!("advisor-{k}"));
4015                if wt.exists() {
4016                    git::worktree_remove(&repo, &wt).await.ok();
4017                }
4018            }
4019        }
4020        if !folded.is_empty() {
4021            self.state
4022                .event("fold", format!("folded candidates {}", folded.join(", ")));
4023            self.state.save()?;
4024        }
4025        Ok(())
4026    }
4027
4028    // ------------------------------------------------------------ base sync
4029
4030    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4031    /// anything verifies it.
4032    ///
4033    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4034    /// read whatever is checked out in the winner's worktree. Left alone that
4035    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4036    /// it when the run *branched* - and a run takes long enough that the base
4037    /// has usually moved by the time it gets here. A gate that ran there
4038    /// answers "green on the commit this run started from", not "green on
4039    /// what is about to land", and the difference showed up three times in
4040    /// one day as a green run whose merge would have reverted a file another
4041    /// pull request had already landed.
4042    ///
4043    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4044    /// `land::Step::Rebase` calls, rather than a second implementation of the
4045    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4046    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4047    /// doc warns about ("two answers to a question nobody notices until a
4048    /// diff is wrong").
4049    ///
4050    /// A conflict is not the end of the road: the standing rebase is handed
4051    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4052    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4053    /// land). Once it finishes, review and the gate run as usual on the
4054    /// rebased tree, which is where a breakage the new base caused is caught
4055    /// by the ordinary gate-fix round. magi resolves nothing itself.
4056    ///
4057    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4058    /// `state.base_sync.attempts`, is how many times the base is *rebased
4059    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4060    /// *conflict* was given to a fixer. When the fixer cannot finish the
4061    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4062    /// what was tried (rounds spent, paths still conflicted) and the branch
4063    /// and worktree stay exactly as they were - untouched, for a person to
4064    /// look at - which is also what makes re-entering this function
4065    /// afterwards a no-op instead of a second attempt at the same wall. A
4066    /// push failure ends the same way.
4067    async fn sync_to_base(&mut self) -> Result<()> {
4068        if self.state.status == RunStatus::AlreadyInBase {
4069            return Ok(());
4070        }
4071        let conflicted = self
4072            .state
4073            .base_sync
4074            .as_ref()
4075            .is_some_and(|s| s.conflict.is_some());
4076        let Some(winner) = self.state.winner().cloned() else {
4077            return Ok(());
4078        };
4079
4080        let repo = self.state.repo.clone();
4081        let remote = self.state.config.merge.remote.clone();
4082        let base_branch = self.state.base_branch.clone();
4083        let tracking = format!("{remote}/{base_branch}");
4084
4085        git::fetch(&repo, &remote, &base_branch).await.ok();
4086        // No network, or the remote never had this branch: the run already
4087        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4088        // otherwise), and one that got this far is not blocked by a fetch
4089        // that fails now. It just has no newer tip to compare against.
4090        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4091            return Ok(());
4092        };
4093
4094        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4095        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4096        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4097
4098        // Before any rebase, and before a recorded conflict is honoured: a
4099        // branch whose change reached the base under other commit ids has
4100        // nothing to rebase and nothing to conflict with, and a run that
4101        // already stopped on that phantom conflict recovers here on resume.
4102        // `behind == 0` with head == tip is a branch the base has since taken
4103        // in whole, whether or not a conflict was ever recorded: the ancestry
4104        // proof must still run (`classify` ignores a head still on the start
4105        // commit).
4106        if (behind > 0 || conflicted || head == tip)
4107            && self
4108                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4109                .await?
4110        {
4111            return Ok(());
4112        }
4113        if conflicted {
4114            return Ok(());
4115        }
4116
4117        if behind == 0 {
4118            // A fixer-finished rebase moves the branch ref before the
4119            // winner's worktree is told (`sync_to_head` below). A run that
4120            // died in between resumes here with `behind == 0` and a tree still
4121            // holding the pre-rebase files, which review and the gate would
4122            // then read. That state is exactly: HEAD moved off the tip the
4123            // rebase started from, yet the tree is still identical to that
4124            // tip. A tree with edits of its own differs from it, so nothing
4125            // is thrown away.
4126            if let Some(from) = self
4127                .state
4128                .rebase_fixes
4129                .iter()
4130                .rev()
4131                .find_map(|r| r.from.clone())
4132                && from != head
4133                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4134                    .await
4135                    .is_ok_and(|o| o.ok())
4136            {
4137                git::sync_to_head(&winner.worktree).await?;
4138            }
4139            // An earlier attempt may have rebased the branch locally and died
4140            // before pushing it (only the fresh-rebase arm below pushes).
4141            // Publish it now, so the plain push at PR time is not refused as
4142            // a non-fast-forward. A run already holding a recorded conflict
4143            // never reaches here; that case is out of scope.
4144            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4145            if let Some(why) = &conflict {
4146                self.state.status = RunStatus::Blocked;
4147                self.state.event("land", why.clone());
4148            }
4149            self.state.base_sync = Some(BaseSync {
4150                tip,
4151                behind: 0,
4152                attempts,
4153                conflict,
4154                already_in: None,
4155            });
4156            self.state.save()?;
4157            return Ok(());
4158        }
4159
4160        if attempts >= BASE_SYNC_ROUNDS {
4161            let why = format!(
4162                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4163                 rebase(s); rebasing again would only race it",
4164                winner.branch
4165            );
4166            self.state.status = RunStatus::Blocked;
4167            self.state.base_sync = Some(BaseSync {
4168                tip,
4169                behind,
4170                attempts,
4171                conflict: Some(why.clone()),
4172                already_in: None,
4173            });
4174            self.state.event("land", why);
4175            self.state.save()?;
4176            return Ok(());
4177        }
4178
4179        self.state.event(
4180            "land",
4181            format!(
4182                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4183                winner.branch
4184            ),
4185        );
4186        self.state.save()?;
4187
4188        // The remote's copy of the branch, read now and only if the fetch
4189        // really succeeded (a stale tracking ref must never pin a lease). It is
4190        // pushed over after a rebase only when it is a commit this branch
4191        // already contains, by ancestry or by patch (an earlier rebase of ours
4192        // that never reached the remote): anything else is somebody else's work.
4193        let branch_tracking = format!("{remote}/{}", winner.branch);
4194        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4195        let remote_tip = if self.state.task_pr_branch().is_none()
4196            && matches!(&fetched_branch, Ok(o) if o.ok())
4197        {
4198            git::rev_parse(&repo, &branch_tracking).await.ok()
4199        } else {
4200            None
4201        };
4202        // A remote tip this branch does not contain is somebody else's work:
4203        // rebasing would leave a local tip that can never be pushed. Stop
4204        // before touching anything and say so.
4205        if let Some(theirs) = &remote_tip
4206            && !git::is_ancestor(&repo, theirs, &head).await
4207            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4208                .await
4209                .is_ok_and(|missing| missing.is_empty())
4210        {
4211            let why = format!(
4212                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4213                 them",
4214                short(theirs),
4215                winner.branch
4216            );
4217            self.state.status = RunStatus::Blocked;
4218            self.state.base_sync = Some(BaseSync {
4219                tip,
4220                behind,
4221                attempts,
4222                conflict: Some(why.clone()),
4223                already_in: None,
4224            });
4225            self.state.event("land", why);
4226            self.state.save()?;
4227            return Ok(());
4228        }
4229
4230        let scratch = self.state.dir().join("base-sync");
4231        let rebased = match crate::rebase::rebase_with_fixer(
4232            &mut self.state,
4233            &scratch,
4234            &winner.branch,
4235            &tracking,
4236        )
4237        .await
4238        {
4239            Ok(crate::rebase::Rebased::Applied) => Ok(None),
4240            Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4241            Err(e) => Err(e),
4242        };
4243        let attempts = attempts + 1;
4244        match rebased {
4245            Ok(None) => {
4246                // The branch ref moved, but a worktree that already had it
4247                // checked out (the winner's) was not told; sync its index and
4248                // files before anything reads them.
4249                git::sync_to_head(&winner.worktree).await?;
4250                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4251                let mut conflict = None;
4252                if let Some(pinned) = &remote_tip {
4253                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4254                    match pushed {
4255                        Ok(o) if o.ok() => self.state.event(
4256                            "land",
4257                            format!("pushed rebased {} to {remote}", winner.branch),
4258                        ),
4259                        Ok(o) => {
4260                            conflict = Some(format!(
4261                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
4262                                winner.branch,
4263                                short(pinned),
4264                                o.stderr.chars().take(600).collect::<String>()
4265                            ));
4266                        }
4267                        Err(e) => {
4268                            conflict = Some(format!(
4269                                "rebased {} locally but could not push it: {e:#}",
4270                                winner.branch
4271                            ));
4272                        }
4273                    }
4274                }
4275                if let Some(why) = &conflict {
4276                    self.state.status = RunStatus::Blocked;
4277                    self.state.event("land", why.clone());
4278                }
4279                self.state.base_sync = Some(BaseSync {
4280                    tip: tip.clone(),
4281                    behind: 0,
4282                    attempts,
4283                    conflict,
4284                    already_in: None,
4285                });
4286                self.state
4287                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
4288            }
4289            Ok(Some(conflict)) => {
4290                let why = format!(
4291                    "{} conflicts with {tracking} and did not rebase: {}",
4292                    winner.branch,
4293                    conflict.chars().take(600).collect::<String>()
4294                );
4295                self.state.status = RunStatus::Blocked;
4296                self.state.base_sync = Some(BaseSync {
4297                    tip,
4298                    behind,
4299                    attempts,
4300                    conflict: Some(why.clone()),
4301                    already_in: None,
4302                });
4303                self.state.event("land", why);
4304            }
4305            Err(e) => {
4306                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4307                self.state.status = RunStatus::Blocked;
4308                self.state.base_sync = Some(BaseSync {
4309                    tip,
4310                    behind,
4311                    attempts,
4312                    conflict: Some(why.clone()),
4313                    already_in: None,
4314                });
4315                self.state.event("land", why);
4316            }
4317        }
4318        self.state.save()?;
4319        Ok(())
4320    }
4321
4322    /// Push a branch an earlier attempt rebased locally but never published,
4323    /// pinned to the remote tip read right after a successful fetch. Returns
4324    /// the reason when the run must stop; `None` when there was nothing to do
4325    /// (no remote copy, the same tip, or a remote copy this branch already
4326    /// contains, which the PR-time push fast-forwards) or the push succeeded.
4327    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4328        // Task competitions publish only after the PR text gate, to their
4329        // public head; never resurrect a candidate branch on the remote.
4330        if self.state.task_pr_branch().is_some() {
4331            return None;
4332        }
4333        let repo = self.state.repo.clone();
4334        let remote = self.state.config.merge.remote.clone();
4335        let fetched = git::fetch(&repo, &remote, branch).await;
4336        if !matches!(&fetched, Ok(o) if o.ok()) {
4337            return None;
4338        }
4339        let branch_tracking = format!("{remote}/{branch}");
4340        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4341        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4342            return None;
4343        }
4344        if !crate::reconcile::origin_missing(&repo, head, &theirs)
4345            .await
4346            .is_ok_and(|missing| missing.is_empty())
4347        {
4348            return Some(format!(
4349                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4350                 them",
4351                short(&theirs)
4352            ));
4353        }
4354        match git::push_pinned(&repo, &remote, branch, &theirs).await {
4355            Ok(o) if o.ok() => {
4356                self.state
4357                    .event("land", format!("pushed rebased {branch} to {remote}"));
4358                None
4359            }
4360            Ok(o) => Some(format!(
4361                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4362                 someone may have pushed): {}",
4363                short(&theirs),
4364                o.stderr.chars().take(600).collect::<String>()
4365            )),
4366            Err(e) => Some(format!(
4367                "{branch} is rebased locally but could not be pushed: {e:#}"
4368            )),
4369        }
4370    }
4371
4372    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
4373    /// is already on `tip` under other commit ids ([`crate::already`]); returns
4374    /// whether it did.
4375    ///
4376    /// Checked only when the base is ahead of the branch. A failing check is
4377    /// "not proven" - the ordinary rebase path then decides - never a reason to
4378    /// stop the run.
4379    ///
4380    /// The remote copy of the branch is held to the same standard as the local
4381    /// one: if it carries a tip this worktree does not, that tip must itself be
4382    /// proven in the base, or nothing is settled (a pull request would
4383    /// otherwise be closed over commits nobody checked). The pull request is
4384    /// closed *before* the terminal status is saved; if that fails for a
4385    /// reason other than a refusal (no network, a `gh` error) the run is left
4386    /// `Blocked` with the reason as its conflict, which a resume retries -
4387    /// the same recovery a phantom conflict gets.
4388    async fn settle_already_in(
4389        &mut self,
4390        branch: &str,
4391        tip: &str,
4392        head: &str,
4393        attempts: usize,
4394        behind: usize,
4395    ) -> Result<bool> {
4396        let repo = self.state.repo.clone();
4397        let remote = self.state.config.merge.remote.clone();
4398        let start = self.state.base_commit.clone();
4399        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4400            Ok(Some(e)) => e,
4401            Ok(None) => return Ok(false),
4402            Err(e) => {
4403                tracing::warn!("already-in-base check for {branch}: {e:#}");
4404                return Ok(false);
4405            }
4406        };
4407        let mut verified = vec![head.to_owned()];
4408        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4409        let fetched = git::fetch(&repo, &remote, &public_branch).await;
4410        if matches!(&fetched, Ok(o) if o.ok())
4411            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4412            && theirs != head
4413        {
4414            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4415                Ok(Some(_)) => verified.push(theirs),
4416                _ => return Ok(false),
4417            }
4418        }
4419        let base_branch = self.state.base_branch.clone();
4420        let message = format!(
4421            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4422             land",
4423            evidence.names(),
4424            evidence.proof.as_str()
4425        );
4426        let closed =
4427            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4428                .await;
4429        match closed {
4430            Ok(Ok(url)) => self
4431                .state
4432                .event("land", format!("closed {url}: superseded on {base_branch}")),
4433            Ok(Err(why)) => self
4434                .state
4435                .event("land", format!("did not close a pull request: {why}")),
4436            Err(e) => {
4437                let why = format!(
4438                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
4439                     not be closed ({e:#}); resume to retry"
4440                );
4441                self.state.status = RunStatus::Blocked;
4442                self.state.base_sync = Some(BaseSync {
4443                    tip: tip.to_owned(),
4444                    behind,
4445                    attempts,
4446                    conflict: Some(why.clone()),
4447                    already_in: None,
4448                });
4449                self.state.event("land", why);
4450                self.state.save()?;
4451                return Ok(true);
4452            }
4453        }
4454        self.state.status = RunStatus::AlreadyInBase;
4455        self.state.base_sync = Some(BaseSync {
4456            tip: tip.to_owned(),
4457            behind,
4458            attempts,
4459            conflict: None,
4460            already_in: Some(evidence),
4461        });
4462        self.state.event("land", message);
4463        self.state.save()?;
4464        self.settle_questions();
4465        Ok(true)
4466    }
4467
4468    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
4469    /// last landed the winner on, once it has run, else the commit the run
4470    /// branched from.
4471    ///
4472    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
4473    /// and `vote` all happen before there is a winner to rebase, so they
4474    /// compare every candidate against the branch point on purpose, and a
4475    /// base that moves after they are already done cannot change an answer
4476    /// they already gave.
4477    fn landing_base(&self) -> String {
4478        self.state
4479            .base_sync
4480            .as_ref()
4481            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
4482    }
4483
4484    // ------------------------------------------------------- operator fix
4485
4486    /// Route specific, already-recorded review findings to a fixer for a
4487    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
4488    /// entry point.
4489    ///
4490    /// Distinct from `review_loop`'s own fix step in three ways: it never
4491    /// runs a reviewer wave, it never spends review-round budget, and what
4492    /// happened is recorded as an [`OperatorFixRequest`] appended to
4493    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
4494    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
4495    /// and vote must never be rewritten to look like a manufactured blocking
4496    /// verdict.
4497    ///
4498    /// Only meaningful once review has actually concluded: `Ready` (handed
4499    /// off with findings still open, or simply concluded clean while minor
4500    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
4501    /// gate failed). Everything else is refused: a run still in progress
4502    /// should simply be resumed, and a `Merged` run's branch has already
4503    /// landed — reopening *this* run's own record cannot change that, so the
4504    /// answer there is a fresh `magi review <branch>`.
4505    ///
4506    /// A real commit here re-verifies through a fresh, ordinary review-only
4507    /// run on the same branch ([`Self::review`]) rather than reopening this
4508    /// run's own `review_loop`: once any round in this run's history went
4509    /// clean, `review_conclusion` treats that as permanent by design (the
4510    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
4511    /// way to force one more genuine reviewer wave out of *this* run without
4512    /// either rewriting history or weakening that guarantee for every other
4513    /// caller. A review-only run costs nothing extra — no implementation, no
4514    /// judging, no vote — and exercises the exact same review → verify →
4515    /// gate → (human) merge path, unmodified.
4516    pub async fn fix_selected(
4517        &mut self,
4518        ids: &[String],
4519        reason: &str,
4520        allow_stale: bool,
4521    ) -> Result<()> {
4522        let reason = reason.trim();
4523        if reason.is_empty() {
4524            bail!("a fix request needs a reason — that is the operator's own record of why");
4525        }
4526        if ids.is_empty() {
4527            bail!("no finding id given");
4528        }
4529        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
4530            bail!(
4531                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
4532                 has already concluded — can be given a targeted fix. A run still \
4533                 in progress should simply be resumed; a `merged` run's branch has \
4534                 already landed, so its answer is a fresh `magi review <branch>`, \
4535                 not reopening this run's own record",
4536                self.state.id,
4537                self.state.status.as_str()
4538            );
4539        }
4540        let Some(winner) = self.state.winner().cloned() else {
4541            bail!("run {} has no winning candidate to fix", self.state.id);
4542        };
4543        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
4544            bail!(
4545                "branch `{}` no longer exists; this run cannot be extended",
4546                winner.branch
4547            );
4548        }
4549        let home = crate::run::home();
4550        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
4551            bail!(
4552                "run {} is currently being worked on by another magi process",
4553                self.state.id
4554            );
4555        }
4556        // Held for the rest of this call, including the follow-up review
4557        // below: two `magi fix` invocations against the same run must not
4558        // both reach the worktree manipulation further down, which would
4559        // otherwise race to remove and recreate the same directory — see
4560        // [`FixClaim`]'s own doc.
4561        let _claim = FixClaim::acquire(&self.state.dir())?;
4562
4563        // Resolve every id before spending anything — an unknown id refuses
4564        // the whole request rather than silently dropping it — and dedup
4565        // while keeping the operator's own order.
4566        let mut seen = BTreeSet::new();
4567        let mut findings = Vec::new();
4568        let mut missing = Vec::new();
4569        for id in ids {
4570            if !seen.insert(id.clone()) {
4571                continue;
4572            }
4573            match self.state.finding(id) {
4574                Some((round, rec, f)) => findings.push(OperatorFixFinding {
4575                    id: f.id.clone(),
4576                    severity: f.severity,
4577                    reviewer_vote: rec.vote,
4578                    round: round.round,
4579                    round_head: round.head.clone(),
4580                    reviewer: rec.reviewer,
4581                    agent: rec.agent.clone(),
4582                    file: f.file.clone(),
4583                    line: f.line,
4584                    title: f.title.clone(),
4585                    detail: f.detail.clone(),
4586                    outcome: OperatorFixOutcome::Pending,
4587                }),
4588                None => missing.push(id.clone()),
4589            }
4590        }
4591        if !missing.is_empty() {
4592            bail!(
4593                "unknown finding id(s): {}; nothing was changed",
4594                missing.join(", ")
4595            );
4596        }
4597
4598        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
4599        let stale_details: Vec<(String, String)> = findings
4600            .iter()
4601            .filter(|f| f.round_head != head_at_request)
4602            .map(|f| (f.id.clone(), f.round_head.clone()))
4603            .collect();
4604        let stale = !stale_details.is_empty();
4605        if stale && !allow_stale {
4606            bail!(
4607                "the branch has moved since some finding(s) were raised — {} — now \
4608                 at {}; pass --allow-stale to fix anyway, or re-run review first",
4609                stale_details
4610                    .iter()
4611                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
4612                    .collect::<Vec<_>>()
4613                    .join(", "),
4614                short(&head_at_request)
4615            );
4616        }
4617
4618        let request = OperatorFixRequest {
4619            requested_at: Timestamp::now(),
4620            reason: reason.to_owned(),
4621            findings,
4622            head_at_request: head_at_request.clone(),
4623            allow_stale,
4624            stale,
4625            fix: None,
4626            result_head: None,
4627            follow_up_review_run: None,
4628        };
4629        self.state.event(
4630            "fix",
4631            format!(
4632                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
4633                request.findings.len(),
4634                request
4635                    .findings
4636                    .iter()
4637                    .map(|f| f.id.as_str())
4638                    .collect::<Vec<_>>()
4639                    .join(", "),
4640            ),
4641        );
4642        // Recorded now, before any worktree work or the fixer call itself —
4643        // and re-saved at each checkpoint below: a crash at any point after
4644        // this (mid fixer call, mid follow-up review) must not lose the fact
4645        // that this was requested, for which findings, and why. Everything
4646        // past this point reads and writes through `request_index` rather
4647        // than a local variable, since `request` itself is moved here.
4648        self.state.operator_fixes.push(request);
4649        self.state.save()?;
4650        let request_index = self.state.operator_fixes.len() - 1;
4651
4652        // A fresh, dedicated worktree for this one call, never the winner's
4653        // own worktree in place: that one may already be gone (folded away),
4654        // and reusing it in place would leave the branch checked out there
4655        // when the follow-up review below tries to check it out again. Freed
4656        // immediately after, either way — but only once confirmed clean:
4657        // `worktree_remove` is a `git worktree remove --force`, which would
4658        // otherwise discard uncommitted work left there by the operator or
4659        // another process before this had a chance to even look at it.
4660        if winner.worktree.exists() {
4661            // Lockfiles a rescue commit withheld stay untracked on purpose and
4662            // are already recorded; they are not the operator's work to protect.
4663            let dirty = git::git(
4664                &winner.worktree,
4665                &["status", "--porcelain", "--untracked-files=all"],
4666            )
4667            .await?;
4668            let only_withheld = dirty.lines().all(|l| {
4669                l.strip_prefix("?? ")
4670                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
4671            });
4672            if !only_withheld {
4673                bail!(
4674                    "`{}` has uncommitted changes; refusing to touch it — commit or \
4675                     discard them first",
4676                    winner.worktree.display()
4677                );
4678            }
4679            git::worktree_remove(&self.state.repo, &winner.worktree)
4680                .await
4681                .ok();
4682        }
4683        let fix_worktree = self.state.worktree_root().join("operator-fix");
4684        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
4685        git::git(
4686            &self.state.repo,
4687            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
4688        )
4689        .await
4690        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
4691        if !git::is_clean(&fix_worktree).await? {
4692            git::worktree_remove(&self.state.repo, &fix_worktree)
4693                .await
4694                .ok();
4695            bail!(
4696                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
4697                winner.branch
4698            );
4699        }
4700        let repo_for_setup = self.state.repo.clone();
4701        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
4702            Ok(()) => {
4703                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
4704            }
4705            Err(e) => Err(e),
4706        };
4707        if let Err(e) = setup {
4708            git::worktree_remove(&self.state.repo, &fix_worktree)
4709                .await
4710                .ok();
4711            return Err(e);
4712        }
4713
4714        let run_id = self.state.id.clone();
4715        let prompts = self.state.config.prompts.clone();
4716        let language = self.state.config.graph.language.clone();
4717        let sessions = self.state.config.graph.sessions;
4718        let artifacts = agent::artifacts_dir(&self.state.dir());
4719        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
4720            .findings
4721            .iter()
4722            .map(|f| Finding {
4723                id: f.id.clone(),
4724                severity: f.severity,
4725                file: f.file.clone(),
4726                line: f.line,
4727                title: f.title.clone(),
4728                detail: f.detail.clone(),
4729            })
4730            .collect();
4731        let fix_prompt = prompt::operator_fix(
4732            &self.state.instruction,
4733            &finding_list,
4734            reason,
4735            &stale_details,
4736            &head_at_request,
4737            &language,
4738        );
4739        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4740        let (job, seat, out) = self
4741            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
4742                prompt: fix_prompt.clone(),
4743                spec,
4744                seat,
4745                cwd: fix_worktree.clone(),
4746                timeout,
4747                allow_write: true,
4748                sessions,
4749                artifacts: artifacts.clone(),
4750                stem: "operator-fix".to_owned(),
4751                handover: None,
4752            })
4753            .await;
4754        let agent_id = seat.agent.clone();
4755
4756        let mut fix = FixRecord {
4757            agent: agent_id,
4758            addressed: Vec::new(),
4759            rejected: Vec::new(),
4760            notes: String::new(),
4761            committed: false,
4762            failed: None,
4763            duration_ms: 0,
4764            continuation: None,
4765        };
4766        let mut final_seat = seat.clone();
4767        match out {
4768            AgentOutcome::Ok(o) => {
4769                fix.duration_ms = o.duration_ms;
4770                let parsed = verdict::extract_json::<FixReport>(&o.text);
4771                let incomplete_reason = match &parsed {
4772                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4773                        "the reply parsed, but it reported a command whose own CLI \
4774                         never confirmed an exit status"
4775                            .to_owned(),
4776                    ),
4777                    Ok(_) => None,
4778                    Err(e) => Some(e.to_string()),
4779                };
4780                match incomplete_reason {
4781                    None => {
4782                        let report = parsed.expect("checked Ok above");
4783                        fix.addressed = report.addressed;
4784                        fix.rejected = report.rejected;
4785                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
4786                    }
4787                    Some(reason) => {
4788                        let (resumed_seat, resolved, failure, cont) = self
4789                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
4790                            .await;
4791                        fix.duration_ms += cont.cumulative_wait_ms;
4792                        fix.continuation = Some(cont);
4793                        final_seat = resumed_seat;
4794                        match resolved {
4795                            Some(report) => {
4796                                fix.addressed = report.addressed;
4797                                fix.rejected = report.rejected;
4798                                fix.notes =
4799                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
4800                            }
4801                            None => fix.failed = failure,
4802                        }
4803                    }
4804                }
4805            }
4806            AgentOutcome::Dropped(o) => {
4807                fix.duration_ms = o.duration_ms;
4808                let why = o
4809                    .dropped
4810                    .as_ref()
4811                    .map(|d| d.why.as_str())
4812                    .unwrap_or("the CLI ended the stream without delivering its answer");
4813                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4814            }
4815            AgentOutcome::Quota(o) => {
4816                self.state.quota.push(QuotaLoss {
4817                    seat: final_seat.key.clone(),
4818                    node: "fix".to_owned(),
4819                    at: Timestamp::now(),
4820                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4821                });
4822                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4823            }
4824            AgentOutcome::Failed(e) => fix.failed = Some(e),
4825            AgentOutcome::ResourceBlocked(e) => {
4826                fix.failed = Some(format!("the fixer was not started: {e}"));
4827            }
4828        }
4829        if fix.continuation.is_none() {
4830            fix.continuation = Some(ContinuationRecord::not_needed());
4831        }
4832        self.state.seats.insert(final_seat.key.clone(), final_seat);
4833
4834        let rescue_message = format!(
4835            "magi: operator-selected fix ({}) (uncommitted work)",
4836            self.state.operator_fixes[request_index]
4837                .findings
4838                .iter()
4839                .map(|f| f.id.as_str())
4840                .collect::<Vec<_>>()
4841                .join(", ")
4842        );
4843        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
4844            self.state.note_withheld("fix", &r.withheld);
4845        }
4846        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
4847        fix.committed = after != head_at_request;
4848        git::worktree_remove(&self.state.repo, &fix_worktree)
4849            .await
4850            .ok();
4851
4852        self.state.event(
4853            "fix",
4854            match &fix.failed {
4855                Some(reason) => format!(
4856                    "operator fix: adoption report was lost ({reason}); {}",
4857                    if fix.committed {
4858                        "committed"
4859                    } else {
4860                        "NO new commit"
4861                    }
4862                ),
4863                None => format!(
4864                    "operator fix: {} addressed, {} rejected, {}",
4865                    fix.addressed.len(),
4866                    fix.rejected.len(),
4867                    if fix.committed {
4868                        "committed"
4869                    } else {
4870                        "NO new commit"
4871                    }
4872                ),
4873            },
4874        );
4875
4876        // Every selected finding gets an outcome — never left `Pending` once
4877        // the fixer's own turn is over. A report that never came back at all
4878        // marks every one of them `Unreported`, not silently "not addressed":
4879        // quota, a dropped stream, or an exhausted continuation are gaps in
4880        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
4881        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
4882        for f in &mut self.state.operator_fixes[request_index].findings {
4883            f.outcome = if fix.failed.is_some() {
4884                OperatorFixOutcome::Unreported
4885            } else if fix.addressed.contains(&f.id) {
4886                OperatorFixOutcome::Addressed
4887            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
4888                OperatorFixOutcome::Rejected { why: r.why.clone() }
4889            } else {
4890                OperatorFixOutcome::Unreported
4891            };
4892        }
4893
4894        let committed = fix.committed;
4895        if committed {
4896            self.state.operator_fixes[request_index].result_head = Some(after.clone());
4897        }
4898        self.state.operator_fixes[request_index].fix = Some(fix);
4899        // Saved again now that the fixer's own outcome is final, on top of
4900        // the save right after the request was first pushed above.
4901        self.state.save()?;
4902
4903        if committed {
4904            self.state.event(
4905                "fix",
4906                format!(
4907                    "operator fix committed {}; opening a follow-up review-only run",
4908                    short(&after)
4909                ),
4910            );
4911            // The operator asked for the fix, and the follow-up serves whatever
4912            // task the run it follows served.
4913            let origin =
4914                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
4915            match Self::review(
4916                &self.state.repo,
4917                &winner.branch,
4918                self.state.config.clone(),
4919                origin,
4920            )
4921            .await
4922            {
4923                Ok(mut follow_up) => {
4924                    follow_up.state.event(
4925                        "start",
4926                        format!(
4927                            "requested by an operator fix on run {} for finding(s) {}",
4928                            self.state.id,
4929                            self.state.operator_fixes[request_index]
4930                                .findings
4931                                .iter()
4932                                .map(|f| f.id.as_str())
4933                                .collect::<Vec<_>>()
4934                                .join(", "),
4935                        ),
4936                    );
4937                    follow_up.state.save()?;
4938                    let follow_up_id = follow_up.state.id.clone();
4939                    // The follow-up is a run like any other: it belongs to the
4940                    // task of the run it follows, or to one filed for it.
4941                    let adopted = match crate::direct::adopt(
4942                        &follow_up.state,
4943                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
4944                    ) {
4945                        Ok(a) => a,
4946                        Err(e) => {
4947                            // An ownerless run must not spend agent calls; it
4948                            // stays saved, and `magi run --resume` adopts it.
4949                            self.state.event(
4950                                "fix",
4951                                format!(
4952                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
4953                                ),
4954                            );
4955                            self.state.save()?;
4956                            return Ok(());
4957                        }
4958                    };
4959                    let executed = follow_up.execute().await;
4960                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
4961                    if let Some(a) = adopted {
4962                        a.finish(&follow_up.state, executed);
4963                    }
4964                    if let Some(e) = failure {
4965                        self.state.event(
4966                            "fix",
4967                            format!(
4968                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
4969                            ),
4970                        );
4971                    }
4972                    self.state.operator_fixes[request_index].follow_up_review_run =
4973                        Some(follow_up_id);
4974                }
4975                Err(e) => {
4976                    self.state.event(
4977                        "fix",
4978                        format!("committed the fix but could not open a follow-up review: {e:#}"),
4979                    );
4980                }
4981            }
4982            self.state.save()?;
4983        }
4984
4985        Ok(())
4986    }
4987
4988    // --------------------------------------------------------------- review
4989
4990    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
4991    /// names, in order, each at most once, moving on only when the call
4992    /// advances (an error, a quota hit or nothing usable - the same decision
4993    /// point `agent::chain_advances` is for the other chained roles).
4994    ///
4995    /// Each agent gets its own seat from `Runner::seat`: the same agent
4996    /// continues its conversation (the winner's own implementer seat when it
4997    /// is the winner's author, now that the competition is over), another
4998    /// takes a fresh one so the full prompt is sent again. A handover is
4999    /// recorded under `node`, which is what makes the fallback stick for the
5000    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5001    /// is returned, so an exhausted chain reads exactly like a single failed
5002    /// fixer: one quota loss, the same wording, the same refund. An earlier
5003    /// attempt's edits are left in the tree and judged, with the final
5004    /// attempt's, by what git says afterwards.
5005    ///
5006    /// Returns the job that produced the outcome, for `continue_fix_report`.
5007    async fn ask_fixer(
5008        &mut self,
5009        winner: &Candidate,
5010        node: &'static str,
5011        round: Option<usize>,
5012        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5013    ) -> (SeatJob, SeatState, AgentOutcome) {
5014        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5015        self.ask_fixer_among(attempts, node, round, None, build)
5016            .await
5017    }
5018
5019    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5020    /// worktree and a commit: before the next agent is asked after one that
5021    /// did not satisfy the call, the worktree is put back on that commit, so
5022    /// a failed agent's partial edits are not carried into the next one.
5023    async fn ask_fixer_among(
5024        &mut self,
5025        attempts: Vec<(AgentSpec, String)>,
5026        node: &'static str,
5027        round: Option<usize>,
5028        restore: Option<(&Path, &str)>,
5029        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5030    ) -> (SeatJob, SeatState, AgentOutcome) {
5031        let run_id = self.state.id.clone();
5032        let prompts = self.state.config.prompts.clone();
5033        let cache = self.state.config.cache_dir();
5034        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5035        let mut last = None;
5036        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5037            let seat = self.seat(&key, &spec.id);
5038            let mut job = build(spec.clone(), seat);
5039            if i > 0 {
5040                job.stem = format!("{}-{}", job.stem, spec.id);
5041            }
5042            let ctx = WaveCtx {
5043                carry_seats: false,
5044                pauses: Some((&self.pause, &self.interrupt)),
5045                run: &run_id,
5046                node,
5047                prompts: &prompts,
5048                cache: cache.as_deref(),
5049                round,
5050            };
5051            let (seat, out) =
5052                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5053            // A resource block never advances the chain: the chosen agent
5054            // was not asked, so it must be the one asked again on resume.
5055            let advances = match &out {
5056                AgentOutcome::Ok(o) => agent::output_advances(o),
5057                AgentOutcome::ResourceBlocked(_) => false,
5058                _ => true,
5059            };
5060            if let Some(next) = ids.get(i + 1)
5061                && advances
5062            {
5063                self.state.seats.insert(seat.key.clone(), seat.clone());
5064                let class =
5065                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5066                let reason = match &out {
5067                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5068                    other => fail_reason(other),
5069                };
5070                record_handover(
5071                    &mut self.state,
5072                    node,
5073                    &seat.key,
5074                    &spec.id,
5075                    next,
5076                    &class,
5077                    &reason,
5078                );
5079                // A failed agent's edits must not reach the next one, and a
5080                // restore that did not work ends the chain: the call fails
5081                // and the caller's own (confirmed) restore decides.
5082                if let Some((wt, rev)) = restore
5083                    && let Err(e) = git::restore_to(wt, rev).await
5084                {
5085                    let why =
5086                        format!("could not restore the worktree before the next fixer: {e:#}");
5087                    tracing::warn!("{why}");
5088                    last = Some((job, seat, AgentOutcome::Failed(why)));
5089                    break;
5090                }
5091                continue;
5092            }
5093            last = Some((job, seat, out));
5094            break;
5095        }
5096        last.expect("the fixer chain always has an entry")
5097    }
5098
5099    /// The one repair a verification-only failure earns (see
5100    /// [`crate::recovery`]), before the first review round of a run that took
5101    /// a branch over.
5102    ///
5103    /// Saved as started *before* the fixer is called, so a resume never
5104    /// repairs twice; a start found on re-entry is rolled back to the commit
5105    /// it began from. A failed or unusable fixer leaves the tree as it was and
5106    /// the review goes on over it, so the first round's own verification
5107    /// reports the failure again. The failure is never carried into a round as
5108    /// a result: verification is re-run on whatever commit this leaves.
5109    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5110        use crate::recovery::{Mode, Repair};
5111        if !self.state.reviews.is_empty() {
5112            return Ok(());
5113        }
5114        let Some(mut rec) = self.state.recovery.clone() else {
5115            return Ok(());
5116        };
5117        if rec.mode != Mode::VerifyRepair {
5118            return Ok(());
5119        }
5120        let wt = winner.worktree.clone();
5121        let head = git::rev_parse(&wt, "HEAD").await?;
5122        match rec.repair.clone() {
5123            Repair::Done { .. } => return Ok(()),
5124            Repair::Started { before } => {
5125                // Interrupted: the unfinished edits are not trusted.
5126                // An unconfirmed rollback blocks the run with `Started` kept,
5127                // so a resume retries it; the tree is never reviewed as is.
5128                if let Err(e) = Self::graft_restore(&wt, &before).await {
5129                    return self
5130                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5131                        .await;
5132                }
5133                rec.repair = Repair::Done {
5134                    before: before.clone(),
5135                    after: before,
5136                    committed: false,
5137                    failed: Some("interrupted; rolled back".to_owned()),
5138                };
5139                self.state
5140                    .event("recovery", "the repair was interrupted and rolled back");
5141                self.state.recovery = Some(rec);
5142                self.state.save()?;
5143                return Ok(());
5144            }
5145            Repair::NotStarted => {}
5146        }
5147        if crate::recovery::recheck(&mut rec, &head) {
5148            self.state.event("recovery", rec.summary());
5149            self.state.recovery = Some(rec);
5150            self.state.save()?;
5151            return Ok(());
5152        }
5153        rec.repair = Repair::Started {
5154            before: head.clone(),
5155        };
5156        self.state.recovery = Some(rec.clone());
5157        self.state.save()?;
5158
5159        let language = self.state.config.graph.language.clone();
5160        let sessions = self.state.config.graph.sessions;
5161        let artifacts = agent::artifacts_dir(&self.state.dir());
5162        let fix_prompt = prompt::recovery_fix(
5163            &self.state.instruction,
5164            &rec,
5165            self.state.config.graph.review_rounds,
5166            &language,
5167        );
5168        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5169        self.state.event(
5170            "recovery",
5171            format!(
5172                "repair: the fixer starts from the failed verification on {}",
5173                short(&head)
5174            ),
5175        );
5176        let (_job, seat, out) = self
5177            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5178                prompt: fix_prompt.clone(),
5179                spec,
5180                seat,
5181                cwd: wt.clone(),
5182                timeout,
5183                allow_write: true,
5184                sessions,
5185                artifacts: artifacts.clone(),
5186                stem: "fix-recovery".to_owned(),
5187                handover: None,
5188            })
5189            .await;
5190        // Never spawned: nothing changed, so the repair goes back to "not
5191        // started" and the run stops resumably, as for a graft.
5192        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5193            if let Err(e) = Self::graft_restore(&wt, &head).await {
5194                return self
5195                    .stop_recovery(rec, &head, format!("resource wait; restore failed: {e:#}"))
5196                    .await;
5197            }
5198            rec.repair = Repair::NotStarted;
5199            self.state.recovery = Some(rec);
5200            self.state.save()?;
5201            return self.stop_for_resource();
5202        }
5203        let failed = match &out {
5204            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5205            AgentOutcome::Quota(o) => {
5206                self.state.quota.push(QuotaLoss {
5207                    seat: seat.key.clone(),
5208                    node: "fix".to_owned(),
5209                    at: Timestamp::now(),
5210                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5211                });
5212                Some("rate limited (quota); the fixer could not run".to_owned())
5213            }
5214            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5215            AgentOutcome::Failed(e) => Some(e.clone()),
5216        };
5217        self.state.seats.insert(seat.key.clone(), seat);
5218        let failed = match failed {
5219            // The agent's partial edits are not kept.
5220            Some(why) => {
5221                if let Err(e) = git::restore_to(&wt, &head).await {
5222                    return self
5223                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5224                        .await;
5225                }
5226                Some(why)
5227            }
5228            None => {
5229                if let Ok(r) = git::rescue_commit(
5230                    &wt,
5231                    "magi: recovery repair of the failed verification (uncommitted work)",
5232                )
5233                .await
5234                {
5235                    self.state.note_withheld("fix", &r.withheld);
5236                }
5237                None
5238            }
5239        };
5240        let after = git::rev_parse(&wt, "HEAD").await?;
5241        self.state.event(
5242            "recovery",
5243            match &failed {
5244                Some(why) => format!("repair: nothing kept ({why})"),
5245                None if after != head => {
5246                    format!("repair: committed {}", short(&after))
5247                }
5248                None => "repair: the fixer made no commit".to_owned(),
5249            },
5250        );
5251        rec.repair = Repair::Done {
5252            before: head.clone(),
5253            committed: after != head,
5254            after,
5255            failed,
5256        };
5257        self.state.recovery = Some(rec);
5258        self.state.save()?;
5259        Ok(())
5260    }
5261
5262    /// A repair whose rollback could not be confirmed: the run is blocked and
5263    /// resumable, and the unconfirmed state is never reviewed.
5264    async fn stop_recovery(
5265        &mut self,
5266        mut rec: crate::recovery::RecoveryRecord,
5267        head: &str,
5268        why: String,
5269    ) -> Result<()> {
5270        // `Started` stays recorded: a resume retries the restore.
5271        rec.repair = crate::recovery::Repair::Started {
5272            before: head.to_owned(),
5273        };
5274        self.state.recovery = Some(rec);
5275        self.state
5276            .event("recovery", format!("repair could not be undone: {why}"));
5277        self.state.status = RunStatus::Blocked;
5278        self.state.save()?;
5279        bail!("recovery repair could not be rolled back: {why}")
5280    }
5281
5282    async fn review_loop(&mut self) -> Result<()> {
5283        // A base that would not rebase is a person's decision, not a review
5284        // round: nothing here would change the answer, and reviewers and a
5285        // fixer would be spending real budget on a tree that cannot land
5286        // regardless of what they find.
5287        if self
5288            .state
5289            .base_sync
5290            .as_ref()
5291            .is_some_and(|s| s.conflict.is_some())
5292        {
5293            return Ok(());
5294        }
5295        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
5296        // agent files with `magi task add` name the run that paid for it. The
5297        // prompt overlay is cloned alongside it because the waves borrow it
5298        // while `self` is mutably borrowed by the node's own bookkeeping.
5299        let run_id = self.state.id.clone();
5300        let prompts = self.state.config.prompts.clone();
5301        let Some(winner) = self.state.winner().cloned() else {
5302            return Ok(());
5303        };
5304        let max_rounds = self.state.config.graph.review_rounds;
5305        // A clean round, an exhausted round budget, or a stalled tree (see
5306        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
5307        // they are recorded — recomputed here, not read off `status`, so a
5308        // reentry into a run that already stopped restates the identical
5309        // verdict instead of silently handing back whatever an earlier node
5310        // in this same walk clobbered `status` to (a solo-candidate
5311        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
5312        // below runs an empty range once the budget is spent, and would
5313        // otherwise fall through without touching `status` at all.
5314        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5315            // A reentry after a crash between the last round's save and
5316            // `stop_reviewing` reaches the hand-off here, not there.
5317            if status == RunStatus::Gating {
5318                self.record_contested_handoff();
5319            }
5320            self.state.status = status;
5321            self.state.save()?;
5322            return Ok(());
5323        }
5324        self.state.status = RunStatus::Reviewing;
5325        // A last recorded round whose own verification never resolved
5326        // (`ResourceBlocked` — the shared build cache, not the patch) is
5327        // never a concluded round, whatever the round budget says: starting
5328        // a fresh round on top of it would spend a whole new reviewer wave
5329        // re-reading an unchanged patch instead of just retrying the one
5330        // check that actually needs it, and once the budget is spent the
5331        // loop below has nothing left to do at all (its range is empty).
5332        // Retry that check directly instead, exactly the same retry
5333        // `stop_reviewing` already does for its own catch-up case.
5334        if self
5335            .state
5336            .reviews
5337            .last()
5338            .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5339        {
5340            let shell = self.state.config.shell();
5341            return self
5342                .stop_reviewing(
5343                    "the last round's own verification never resolved",
5344                    &shell,
5345                    &winner.worktree,
5346                )
5347                .await;
5348        }
5349
5350        self.recovery_repair(&winner).await?;
5351        if self.resource_halt()? {
5352            return Ok(());
5353        }
5354
5355        let repo = self.state.repo.clone();
5356        let root = self.state.worktree_root();
5357        let language = self.state.config.graph.language.clone();
5358        let sessions = self.state.config.graph.sessions;
5359        let artifacts = agent::artifacts_dir(&self.state.dir());
5360        let base = self.landing_base();
5361        let base_short = short(&base);
5362        let reviewers = self.roles.reviewers.clone();
5363        let shell = self.state.config.shell();
5364
5365        for round in (self.state.reviews.len() + 1)..=max_rounds {
5366            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5367            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5368            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5369            // The prior round's own record, already persisted — never a
5370            // hand-carried variable of just its failing output: that is
5371            // exactly what let a round's e2e result drift out of sync with
5372            // which commit it was actually about (see `SCHEMA`'s doc for
5373            // schema 8). Judged against `head`, the commit reviewers are
5374            // about to look at now, so the summary always reads as "an
5375            // earlier head" here — this round's own patch has not been
5376            // checked yet.
5377            let prev_verification = self
5378                .state
5379                .reviews
5380                .last()
5381                .and_then(|r| r.verification_summary(&head));
5382            // Only the first round of a run that took a branch over: later
5383            // rounds have this run's own history.
5384            let prior_section = match (&self.state.recovery, round) {
5385                (Some(rec), 1) if self.state.reviews.is_empty() => {
5386                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5387                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5388                            Some(crate::recovery::head_of(
5389                                &git::diff(&winner.worktree, reviewed, "HEAD")
5390                                    .await
5391                                    .unwrap_or_default(),
5392                                crate::recovery::SINCE_DIFF_MAX,
5393                            ))
5394                        }
5395                        _ => None,
5396                    };
5397                    Some(prompt::recovery_review(rec, since.as_deref()))
5398                }
5399                _ => None,
5400            };
5401
5402            // Each reviewer gets its own detached checkout of exactly this
5403            // commit: nobody can perturb the winner's tree, and the fixer can
5404            // keep working without racing a reviewer.
5405            let mut jobs = Vec::new();
5406            for (r, spec) in reviewers.iter().cloned().enumerate() {
5407                let wt = root.join(format!("review-{}", r + 1));
5408                // `clean -fdx` in a reset wipes setup's products, so every
5409                // round sets the seat up again.
5410                if wt.exists() {
5411                    git::reset_detached(&wt, &head).await?;
5412                } else {
5413                    git::worktree_add_detached(&repo, &wt, &head).await?;
5414                }
5415                self.setup_seat_worktree(&repo, &wt, false).await?;
5416                let seat_key = format!("review-{}", r + 1);
5417                // The seat starts the round on whoever answered it last, not
5418                // on the agent the spec names, so a failure is not re-paid.
5419                let spec = pick_start_spec(
5420                    &self.roles.reviewer_roster,
5421                    spec,
5422                    self.state.seat_history.get(&seat_key),
5423                );
5424                let seat = self.seat(&seat_key, &spec.id);
5425                jobs.push(SeatJob {
5426                    prompt: prompt::review(&prompt::ReviewCtx {
5427                        instruction: &self.state.instruction,
5428                        branch: &winner.branch,
5429                        base_short: &base_short,
5430                        stat: &stat,
5431                        patch: &patch,
5432                        verification: prev_verification.as_ref(),
5433                        prior: prior_section.as_deref(),
5434                        reviewers: reviewers.len(),
5435                        round,
5436                        rounds: max_rounds,
5437                        // A review-only run has no rankings, so nothing
5438                        // competed for this patch and the reviewer is told so.
5439                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5440                        lens: Lens::for_seat(r),
5441                        language: &language,
5442                    }),
5443                    spec,
5444                    seat,
5445                    cwd: wt,
5446                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5447                    allow_write: false,
5448                    sessions,
5449                    artifacts: artifacts.clone(),
5450                    stem: format!("review-{round}-{}", r + 1),
5451                    handover: None,
5452                });
5453            }
5454
5455            self.state.event(
5456                "review",
5457                format!(
5458                    "round {round}: {} reviewers on {}",
5459                    jobs.len(),
5460                    short(&head)
5461                ),
5462            );
5463            let mut quota_losses = Vec::new();
5464            let review_retries = self.state.config.graph.retries;
5465            let review_cache = self.state.config.cache_dir();
5466            let ctx = WaveCtx {
5467                carry_seats: true,
5468                pauses: Some((&self.pause, &self.interrupt)),
5469                run: &run_id,
5470                node: "review",
5471                prompts: &prompts,
5472                cache: review_cache.as_deref(),
5473                round: Some(round),
5474            };
5475            let results = ask_json_wave::<Review>(
5476                jobs,
5477                Arc::clone(&self.sem),
5478                review_retries,
5479                &self.roles.reviewer_roster,
5480                &ctx,
5481                &mut quota_losses,
5482                &mut self.state,
5483                &|_: &Review| Ok(()),
5484            )
5485            .await;
5486            if self.state.resource_block.is_some() {
5487                return self.stop_for_resource();
5488            }
5489            // Counted before the move below: how many of *this* round's
5490            // reviewer seats were lost to their own rate limit, as opposed to
5491            // a crash, a timeout, or unparsable output — see `round_is_clean`.
5492            let round_quota_missing = quota_losses.len();
5493            self.state.quota.extend(quota_losses);
5494
5495            let mut records = Vec::new();
5496            let mut all_findings = Vec::new();
5497            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
5498                let agent_id = seat.agent.clone();
5499                self.state.seats.insert(seat.key.clone(), seat);
5500                let mut record = ReviewRecord {
5501                    reviewer: r + 1,
5502                    agent: agent_id,
5503                    summary: String::new(),
5504                    findings: Vec::new(),
5505                    vote: None,
5506                    failed: None,
5507                    duration_ms: 0,
5508                    // Set for both outcomes: `failed: Some(_)` with
5509                    // `attempts > 0` is a seat every retry still lost, not a
5510                    // recovered one — only `failed: None` with `attempts > 0`
5511                    // reads as "answered after a nudge" (see this field's own
5512                    // doc).
5513                    attempts,
5514                };
5515                match res {
5516                    Ok((review, out)) => {
5517                        // Sanitized here, at the point every other piece of
5518                        // agent prose in this file is (candidate summaries,
5519                        // deliberation turns, vote reasons): a reviewer's own
5520                        // words are the one thing about it that could name
5521                        // it, and reconsideration below broadcasts this same
5522                        // summary and these same findings to every other
5523                        // seat on the panel.
5524                        record.summary =
5525                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
5526                        record.vote = Some(review.vote);
5527                        record.duration_ms = out.duration_ms;
5528                        for (n, mut f) in review.findings.into_iter().enumerate() {
5529                            // ids are magi's, never the agent's: the fixer's
5530                            // adoption report is keyed by them.
5531                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
5532                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
5533                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
5534                            // `file` is agent-supplied prose too, never
5535                            // checked against the real tree — the same
5536                            // exposure `title`/`detail` above have, just in
5537                            // a field easy to forget because it looks like a
5538                            // path rather than free text.
5539                            f.file = f
5540                                .file
5541                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
5542                            all_findings.push(f.clone());
5543                            record.findings.push(f);
5544                        }
5545                        self.state.event(
5546                            "review",
5547                            format!(
5548                                "round {round}: reviewer {} voted {} with {} finding(s)",
5549                                r + 1,
5550                                review.vote.label(),
5551                                record.findings.len()
5552                            ),
5553                        );
5554                    }
5555                    Err(e) => {
5556                        record.failed = Some(e.to_string());
5557                        self.state.event(
5558                            "review",
5559                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
5560                        );
5561                    }
5562                }
5563                records.push(record);
5564            }
5565
5566            // Tally the round's votes and, if they split, spend the one
5567            // round of reconsideration the split -> deliberate -> revote
5568            // shape `judge`/`vote` use for the panel, sized down to what a
5569            // read-only review round can afford: one round, and a revote
5570            // rather than an argument, because the panel already wrote its
5571            // reasoning down as findings the first time around.
5572            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
5573            let vote_split =
5574                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
5575            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
5576            if vote_split {
5577                self.state.event(
5578                    "review",
5579                    format!(
5580                        "round {round}: votes split ({}) — one round of reconsideration",
5581                        initial_votes
5582                            .iter()
5583                            .map(|v| v.label())
5584                            .collect::<Vec<_>>()
5585                            .join(", ")
5586                    ),
5587                );
5588                // Seats read every seat's findings and votes, still numbered
5589                // and never named — the same anonymity `review` itself keeps.
5590                let panel: Vec<ReviewSeatReport<'_>> = records
5591                    .iter()
5592                    .filter_map(|r| {
5593                        r.vote.map(|vote| ReviewSeatReport {
5594                            reviewer: r.reviewer,
5595                            vote,
5596                            summary: &r.summary,
5597                            findings: &r.findings,
5598                        })
5599                    })
5600                    .collect();
5601
5602                let mut jobs = Vec::new();
5603                let mut seats_at = Vec::new();
5604                for (r, spec) in reviewers.iter().cloned().enumerate() {
5605                    // A seat with no initial vote has nothing to reconsider
5606                    // from and stays absent, the same as it stayed absent
5607                    // from `panel` above.
5608                    if records[r].vote.is_none() {
5609                        continue;
5610                    }
5611                    let wt = root.join(format!("review-{}", r + 1));
5612                    let seat_key = format!("review-{}", r + 1);
5613                    let spec = self.occupant(&seat_key, spec);
5614                    let seat = self.seat(&seat_key, &spec.id);
5615                    // A seat with no live session has already forgotten the
5616                    // initial review's prompt — restate the patch it is
5617                    // voting on, the same as `deliberate`/`vote` do for a
5618                    // judge in the same position.
5619                    // The panel already carries this seat's own review and
5620                    // vote, so restating the patch makes the prompt whole for
5621                    // a seat handed to another agent.
5622                    let build = |with_patch: bool| {
5623                        prompt::review_reconsider(&ReviewReconsiderCtx {
5624                            instruction: &self.state.instruction,
5625                            reviewer: r + 1,
5626                            lens: Lens::for_seat(r),
5627                            panel: &panel,
5628                            patch: with_patch.then_some(ReviewPatch {
5629                                branch: &winner.branch,
5630                                base_short: &base_short,
5631                                stat: &stat,
5632                                patch: &patch,
5633                            }),
5634                            round,
5635                            rounds: max_rounds,
5636                            language: &language,
5637                        })
5638                    };
5639                    let full = build(true);
5640                    let prompt = if has_context(&spec, &seat, sessions) {
5641                        build(false)
5642                    } else {
5643                        full.clone()
5644                    };
5645                    jobs.push(SeatJob {
5646                        prompt,
5647                        spec,
5648                        seat,
5649                        cwd: wt,
5650                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5651                        allow_write: false,
5652                        sessions,
5653                        artifacts: artifacts.clone(),
5654                        stem: format!("review-{round}-reconsider-{}", r + 1),
5655                        handover: Some(full),
5656                    });
5657                    seats_at.push(r);
5658                }
5659
5660                let mut recon_quota_losses = Vec::new();
5661                let recon_cache = self.state.config.cache_dir();
5662                let recon_ctx = WaveCtx {
5663                    carry_seats: true,
5664                    pauses: Some((&self.pause, &self.interrupt)),
5665                    run: &run_id,
5666                    node: "review",
5667                    prompts: &prompts,
5668                    cache: recon_cache.as_deref(),
5669                    round: Some(round),
5670                };
5671                let recon_results = ask_json_wave::<ReviewRevote>(
5672                    jobs,
5673                    Arc::clone(&self.sem),
5674                    review_retries,
5675                    &self.roles.reviewer_roster,
5676                    &recon_ctx,
5677                    &mut recon_quota_losses,
5678                    &mut self.state,
5679                    &|_: &ReviewRevote| Ok(()),
5680                )
5681                .await;
5682                if self.state.resource_block.is_some() {
5683                    return self.stop_for_resource();
5684                }
5685                self.state.quota.extend(recon_quota_losses);
5686
5687                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
5688                    let agent_id = seat.agent.clone();
5689                    self.state.seats.insert(seat.key.clone(), seat);
5690                    let mut rec = ReviewRevoteRecord {
5691                        reviewer: r + 1,
5692                        agent: agent_id,
5693                        vote: None,
5694                        reason: String::new(),
5695                        failed: None,
5696                    };
5697                    match res {
5698                        Ok((rv, _)) => {
5699                            rec.vote = Some(rv.vote);
5700                            rec.reason =
5701                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
5702                            self.state.event(
5703                                "review",
5704                                format!(
5705                                    "round {round}: reviewer {} revoted {}",
5706                                    r + 1,
5707                                    rv.vote.label()
5708                                ),
5709                            );
5710                        }
5711                        Err(e) => {
5712                            rec.failed = Some(e.to_string());
5713                            self.state.event(
5714                                "review",
5715                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
5716                            );
5717                        }
5718                    }
5719                    reconsideration.push(rec);
5720                }
5721            } else if initial_votes.len() > 1 {
5722                self.state.event(
5723                    "review",
5724                    format!(
5725                        "round {round}: votes agreed ({}) — no reconsideration",
5726                        initial_votes[0].label()
5727                    ),
5728                );
5729            }
5730
5731            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
5732            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5733            // A round that already has a blocking finding and a round left to
5734            // try is going back to the fixer no matter what `verify.e2e`
5735            // says, so running it first only spends the loop's slowest step
5736            // (minutes, for a Rust repo's full test suite) on a head about
5737            // to be rewritten. Deferred, never skipped: `verify.e2e` still
5738            // runs once a round has no blocking findings left (see
5739            // `round_is_clean`, which a deferred — empty — `e2e` can never
5740            // satisfy since `blocking` is nonzero whenever this branch is
5741            // taken), and `stop_reviewing` forces a real run before it will
5742            // ever read a deferred round as green.
5743            let defer_e2e =
5744                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
5745            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
5746                let reason =
5747                    format!("{blocking} blocking finding(s) already required a fix this round");
5748                self.state.event(
5749                    "verify",
5750                    format!(
5751                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
5752                         {}); it will run once a round has none left",
5753                        short(&head)
5754                    ),
5755                );
5756                (Vec::new(), false, true, Some(reason))
5757            } else {
5758                let e2e_commands = self.state.config.verify.e2e.clone();
5759                let cache_dir = self.state.config.cache_dir();
5760                let context = format!("round {round}");
5761                let (e2e, verify_retried) = with_cache_lease(
5762                    &mut self.state,
5763                    cache_dir.as_deref(),
5764                    "e2e",
5765                    "e2e",
5766                    &winner.worktree,
5767                    &head,
5768                    verify_timeout,
5769                    &context,
5770                    |state, budget| {
5771                        let shell = shell.clone();
5772                        let e2e_commands = e2e_commands.clone();
5773                        let worktree = winner.worktree.clone();
5774                        let context = context.clone();
5775                        async move {
5776                            run_e2e_with_retry(
5777                                state,
5778                                &shell,
5779                                &e2e_commands,
5780                                &worktree,
5781                                budget,
5782                                &context,
5783                            )
5784                            .await
5785                        }
5786                    },
5787                )
5788                .await;
5789                (e2e, verify_retried, false, None)
5790            };
5791
5792            let expected = records.len();
5793            let answered = records.iter().filter(|r| r.failed.is_none()).count();
5794            let incomplete = answered < expected;
5795            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
5796            let policy = self.state.config.graph.incomplete_review;
5797            let clean = round_is_clean(
5798                blocking,
5799                e2e_ok,
5800                answered,
5801                expected,
5802                round_quota_missing,
5803                policy,
5804            );
5805
5806            let mut round_record = ReviewRound {
5807                round,
5808                head: head.clone(),
5809                verified_head: None,
5810                verified_at: None,
5811                reviews: records,
5812                e2e,
5813                verify_retried,
5814                e2e_deferred,
5815                e2e_defer_reason,
5816                fix: None,
5817                blocking,
5818                answered,
5819                expected,
5820                clean,
5821                progressed: false,
5822                vote_split,
5823                reconsideration,
5824                verdict: None,
5825            };
5826            // The final vote per seat is its revote where reconsideration
5827            // ran and answered, its initial vote otherwise — the same
5828            // fallback `tally` uses for a judge whose private vote failed.
5829            round_record.verdict = ReviewVote::worst(
5830                round_record
5831                    .final_votes()
5832                    .into_iter()
5833                    .map(|(_, _, vote)| vote),
5834            );
5835            // Which commit and when magi actually attempted to check —
5836            // known the moment a command was dispatched against `head`,
5837            // whether or not it finished: a resource-blocked attempt still
5838            // targeted a specific commit at a specific time, and leaving
5839            // that unrecorded is exactly what made `verification_summary`
5840            // report a fresh attempt as "commit unknown ... recorded before
5841            // this was tracked", indistinguishable from a genuinely old,
5842            // untracked record. Only a deferred or unconfigured round never
5843            // ran at all and has nothing to record — see
5844            // `ReviewRound::verified_head`'s own doc.
5845            if !matches!(
5846                round_record.e2e_status(),
5847                E2eStatus::Deferred | E2eStatus::NotConfigured
5848            ) {
5849                round_record.verified_head = Some(head.clone());
5850                round_record.verified_at = Some(Timestamp::now());
5851            }
5852            let this_round_verification = round_record.verification_summary(&head);
5853
5854            if incomplete {
5855                let missing: Vec<String> = round_record
5856                    .reviews
5857                    .iter()
5858                    .filter(|r| r.failed.is_some())
5859                    .map(|r| format!("review-{}", r.reviewer))
5860                    .collect();
5861                self.state.event(
5862                    "review",
5863                    format!(
5864                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
5865                        missing.join(", ")
5866                    ),
5867                );
5868            }
5869
5870            if clean {
5871                self.state.event(
5872                    "review",
5873                    if incomplete && policy == IncompleteReviewPolicy::Warn {
5874                        format!(
5875                            "round {round}: clean (warn policy, incomplete panel) — no \
5876                             blocking findings from the seats that answered, verification green"
5877                        )
5878                    } else if incomplete {
5879                        format!(
5880                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
5881                             quorum) — no blocking findings from the seats that answered, \
5882                             verification green",
5883                            expected - answered
5884                        )
5885                    } else {
5886                        format!("round {round}: clean — no blocking findings, verification green")
5887                    },
5888                );
5889                self.state.reviews.push(round_record);
5890                self.state.status = RunStatus::Gating;
5891                self.state.save()?;
5892                return Ok(());
5893            }
5894
5895            // Nothing was raised and verification passed, but not every seat
5896            // answered and `round_is_clean` still refused to call it clean —
5897            // either a seat is missing for a reason other than its own quota
5898            // (a crash, a timeout, unparsable output — worth another try), or
5899            // every seat that could have answered lost its quota and nobody
5900            // is left to decide on: re-review rather than send the fixer
5901            // after a round with nothing to fix.
5902            if incomplete && blocking == 0 && e2e_ok {
5903                self.state.reviews.push(round_record);
5904                self.state.save()?;
5905                if round == max_rounds {
5906                    self.state.status = RunStatus::Blocked;
5907                    self.state.event(
5908                        "review",
5909                        format!(
5910                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
5911                             refusing to call it clean",
5912                            expected - answered
5913                        ),
5914                    );
5915                    return Ok(());
5916                }
5917                continue;
5918            }
5919
5920            // Nothing for the fixer to act on (`blocking == 0`) and the only
5921            // reason this round is not clean is that magi itself never got
5922            // a command to run — the shared build cache, not the patch (see
5923            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
5924            // the fixer would invite a change to appease contention that has
5925            // nothing to do with the diff, and would leave this attempt
5926            // sitting in the next round's prompt as if it were about an
5927            // earlier, superseded commit rather than what it actually is:
5928            // the same head, still waiting to be checked. Wait for it the
5929            // same way the final round's own contention is already handled,
5930            // whatever round this happens to be.
5931            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
5932                self.state.reviews.push(round_record);
5933                return self
5934                    .stop_reviewing(
5935                        "the round's own verification could not run",
5936                        &shell,
5937                        &winner.worktree,
5938                    )
5939                    .await;
5940            }
5941
5942            if round == max_rounds {
5943                self.state.reviews.push(round_record);
5944                return self
5945                    .stop_reviewing(
5946                        &format!(
5947                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
5948                        ),
5949                        &shell,
5950                        &winner.worktree,
5951                    )
5952                    .await;
5953            }
5954
5955            // Fix. The winner's own implementer seat continues its conversation:
5956            // the competition is over, so context is pure benefit now.
5957            let blocking_findings: Vec<_> = all_findings
5958                .iter()
5959                .filter(|f| f.severity.blocks())
5960                .cloned()
5961                .collect();
5962            let fix_prompt = prompt::fix(
5963                &self.state.instruction,
5964                &blocking_findings,
5965                this_round_verification.as_ref(),
5966                round,
5967                max_rounds,
5968                &language,
5969            );
5970            let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5971            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
5972            let (job, seat, out) = self
5973                .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
5974                    prompt: fix_prompt.clone(),
5975                    spec,
5976                    seat,
5977                    cwd: winner.worktree.clone(),
5978                    timeout,
5979                    allow_write: true,
5980                    sessions,
5981                    artifacts: artifacts.clone(),
5982                    stem: format!("fix-{round}"),
5983                    handover: None,
5984                })
5985                .await;
5986            // Not asked: the round is not recorded (so a resume repeats it),
5987            // no fixer was handed over to, and the seat is left as it was.
5988            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5989                return self.stop_for_resource();
5990            }
5991            let agent_id = seat.agent.clone();
5992
5993            let mut fix = FixRecord {
5994                agent: agent_id,
5995                addressed: Vec::new(),
5996                rejected: Vec::new(),
5997                notes: String::new(),
5998                committed: false,
5999                failed: None,
6000                duration_ms: 0,
6001                continuation: None,
6002            };
6003            let mut continuation = ContinuationRecord::not_needed();
6004            let mut final_seat = seat.clone();
6005            match out {
6006                AgentOutcome::Ok(o) => {
6007                    fix.duration_ms = o.duration_ms;
6008                    let parsed = verdict::extract_json::<FixReport>(&o.text);
6009                    // A parsed report standing next to a command this same
6010                    // reply's own CLI never confirmed the exit status of is
6011                    // not a resolved answer — the identical `CommandEvidence`
6012                    // `state.jobs` renders, read here instead of only on
6013                    // display, per the completion judgment and the shown
6014                    // record needing to agree.
6015                    let incomplete_reason = match &parsed {
6016                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6017                            "the reply parsed, but it reported a command whose own CLI never \
6018                             confirmed an exit status"
6019                                .to_owned(),
6020                        ),
6021                        Ok(_) => None,
6022                        Err(e) => Some(e.to_string()),
6023                    };
6024                    match incomplete_reason {
6025                        None => {
6026                            let report = parsed.expect("checked Ok above");
6027                            fix.addressed = report.addressed;
6028                            fix.rejected = report.rejected;
6029                            fix.notes =
6030                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
6031                        }
6032                        Some(reason) => {
6033                            let (resumed_seat, resolved, failure, cont) = self
6034                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6035                                .await;
6036                            fix.duration_ms += cont.cumulative_wait_ms;
6037                            continuation = cont;
6038                            final_seat = resumed_seat;
6039                            match resolved {
6040                                Some(report) => {
6041                                    fix.addressed = report.addressed;
6042                                    fix.rejected = report.rejected;
6043                                    fix.notes = blind::sanitize_prose(
6044                                        &report.notes,
6045                                        &self.state.config.blind,
6046                                    );
6047                                }
6048                                None => fix.failed = failure,
6049                            }
6050                        }
6051                    }
6052                }
6053                // The CLI's raw error JSON is not a fix report to parse.
6054                AgentOutcome::Dropped(o) => {
6055                    fix.duration_ms = o.duration_ms;
6056                    let why = o
6057                        .dropped
6058                        .as_ref()
6059                        .map(|d| d.why.as_str())
6060                        .unwrap_or("the CLI ended the stream without delivering its answer");
6061                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6062                }
6063                AgentOutcome::Quota(o) => {
6064                    self.state.quota.push(QuotaLoss {
6065                        seat: final_seat.key.clone(),
6066                        node: "fix".to_owned(),
6067                        at: Timestamp::now(),
6068                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6069                    });
6070                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6071                }
6072                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6073            }
6074            fix.continuation = Some(continuation);
6075            self.state.seats.insert(final_seat.key.clone(), final_seat);
6076            if let Ok(r) = git::rescue_commit(
6077                &winner.worktree,
6078                &format!("magi: review round {round} fixes (uncommitted work)"),
6079            )
6080            .await
6081            {
6082                self.state.note_withheld("fix", &r.withheld);
6083            }
6084            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6085            fix.committed = after != before;
6086            // Judged by what `git` says moved against base, never by the
6087            // fixer's own `addressed`/`rejected` count — see
6088            // `ReviewRound::progressed`. Propagated with `?`, the same as the
6089            // `patch` snapshot above: swallowing this error would default
6090            // `diff_after` to empty, which almost always differs from a
6091            // non-empty `patch` and reads as "progressed" — exactly backwards
6092            // for a `git` failure the stagnation check cannot see through.
6093            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6094            let progressed = diff_after != patch;
6095            let commit_note = if fix.committed {
6096                "committed"
6097            } else {
6098                "NO new commit"
6099            };
6100            let tree_note = if progressed {
6101                "changed vs base"
6102            } else {
6103                "unchanged vs base"
6104            };
6105            self.state.event(
6106                "fix",
6107                match &fix.failed {
6108                    // Distinct on purpose from "0 addressed, 0 rejected": the
6109                    // fixer's own diff still landed (blocking counts do keep
6110                    // falling round over round), only its adoption report did
6111                    // not come back, so this must never read like every
6112                    // finding was reviewed and declined.
6113                    Some(reason) => {
6114                        format!(
6115                            "round {round}: fixer's adoption report was lost ({reason}); \
6116                             {commit_note}, tree {tree_note}"
6117                        )
6118                    }
6119                    None => format!(
6120                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6121                         {tree_note}{}",
6122                        fix.addressed.len(),
6123                        fix.rejected.len(),
6124                        if continuation.outcome == ContinuationOutcome::Resumed {
6125                            format!(
6126                                " (adoption report recovered after {} continuation(s))",
6127                                continuation.attempts
6128                            )
6129                        } else {
6130                            String::new()
6131                        },
6132                    ),
6133                },
6134            );
6135            round_record.fix = Some(fix);
6136            round_record.progressed = progressed;
6137            self.state.reviews.push(round_record);
6138            self.state.save()?;
6139
6140            // The fixer's own report never came back this round, even after
6141            // `continue_fix_report`'s own budget was spent on it — not an
6142            // ordinary "no report" (dropped stream, quota, plain failure),
6143            // which already reads that way and is left to the existing round
6144            // budget. Stopping here, rather than opening another round, is
6145            // what keeps a next reviewer/fixer wave from ever being
6146            // dispatched onto `winner.worktree` while whatever the seat's
6147            // last call may still have running there is unaccounted for: no
6148            // process liveness check exists (and none is being added — see
6149            // AGENTS.md/this task's own scope), so the only way to honour
6150            // "nothing starts before a valid report returns" is to not start
6151            // anything further on this worktree from this run at all.
6152            if matches!(
6153                continuation.outcome,
6154                ContinuationOutcome::Exhausted
6155                    | ContinuationOutcome::QuotaLost
6156                    | ContinuationOutcome::NoSession
6157            ) {
6158                return self
6159                    .stop_reviewing(
6160                        "the fixer's adoption report never came back, even after resuming its \
6161                         own seat; refusing to start another round against the same worktree \
6162                         while that is unresolved",
6163                        &shell,
6164                        &winner.worktree,
6165                    )
6166                    .await;
6167            }
6168
6169            let streak = self
6170                .state
6171                .reviews
6172                .iter()
6173                .rev()
6174                .take_while(|r| !r.progressed)
6175                .count();
6176            if streak >= STAGNANT_LIMIT {
6177                return self
6178                    .stop_reviewing(
6179                        &format!(
6180                            "the tree has not moved against base for {streak} round(s) in a row"
6181                        ),
6182                        &shell,
6183                        &winner.worktree,
6184                    )
6185                    .await;
6186            }
6187        }
6188        Ok(())
6189    }
6190
6191    /// Decide, from the last recorded round's own verification, whether
6192    /// stopping the review loop is a hand-off or a genuine block.
6193    ///
6194    /// Called once the loop has given up trying — the round budget is spent,
6195    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
6196    /// findings still open, never while a round is still clean or the
6197    /// incomplete-panel case handled inline above. Gate and e2e are facts
6198    /// about the tree; a lingering review finding is an opinion, and this
6199    /// workload's own `magi stats` puts reviewer precision low enough
6200    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
6201    /// must not by itself stand between a green, verified change and the
6202    /// human who decides what to do with it. A red e2e is not an opinion, so
6203    /// that case still blocks, with the failing command and a tail of its
6204    /// output recorded here rather than left in `run.json` for someone to go
6205    /// find.
6206    ///
6207    /// A round that deferred its own e2e (see [`Config::graph`]'s
6208    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
6209    /// only because nothing ran, and treating an empty list as a passing one
6210    /// here is exactly the "deferred painted green" bug this function exists
6211    /// to not have. When the last round's own verification never resolved —
6212    /// deferred on purpose, or a real attempt the shared build cache blocked
6213    /// — this makes (or retries) the real run, on the actual worktree this
6214    /// loop is about to stop touching, before deciding anything. A
6215    /// resource-blocked attempt is likewise never read as either green or
6216    /// red: it is evidence about the machine, not the patch (see
6217    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
6218    /// blocked cache leaves this call without deciding rather than guessing
6219    /// — the caller retries on a later reentry.
6220    /// Record, once, that the review loop handed off over a blocking finding
6221    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
6222    /// `land` asks the owner even with `land_approval` off. Called from every
6223    /// path that concludes `Gating`; a reentry keeps the first record.
6224    fn record_contested_handoff(&mut self) {
6225        if self.state.contested_handoff.is_some() {
6226            return;
6227        }
6228        let Some(contested) = self
6229            .state
6230            .reviews
6231            .last()
6232            .and_then(ReviewRound::contested_handoff)
6233        else {
6234            return;
6235        };
6236        self.state.event(
6237            "review",
6238            format!(
6239                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6240                 wait for the owner's approval",
6241                contested.findings.len(),
6242                contested.rejecters.len()
6243            ),
6244        );
6245        self.state.contested_handoff = Some(contested);
6246    }
6247
6248    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6249        let round_idx = self.state.reviews.len() - 1;
6250        // A deferred round and a resource-blocked one are the same shape
6251        // here: neither has a real result yet, and both get one more
6252        // attempt. Read off `e2e_status` — the single source for this —
6253        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
6254        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
6255        // still retries instead of being read as a settled result the
6256        // instant it stops being empty.
6257        let needs_catchup_run = matches!(
6258            self.state.reviews[round_idx].e2e_status(),
6259            E2eStatus::Deferred | E2eStatus::ResourceBlocked
6260        );
6261        if needs_catchup_run {
6262            let round = self.state.reviews[round_idx].round;
6263            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6264            let commands = self.state.config.verify.e2e.clone();
6265            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6266            let cache_dir = self.state.config.cache_dir();
6267            let context = format!(
6268                "round {round}: verification unresolved, catching up before the final decision"
6269            );
6270            let (outcomes, verify_retried) = with_cache_lease(
6271                &mut self.state,
6272                cache_dir.as_deref(),
6273                "e2e",
6274                "e2e",
6275                worktree,
6276                &attempted_head,
6277                timeout,
6278                &context,
6279                |state, budget| {
6280                    let shell = shell.to_vec();
6281                    let commands = commands.clone();
6282                    let context = context.clone();
6283                    async move {
6284                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6285                            .await
6286                    }
6287                },
6288            )
6289            .await;
6290            let last = &mut self.state.reviews[round_idx];
6291            last.e2e = outcomes;
6292            last.verify_retried = verify_retried;
6293            // Always the commit and time this attempt actually targeted,
6294            // whether or not it happens to equal the reviewed `head` and
6295            // whether or not a command finished — see
6296            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
6297            // attempt is recorded too, so a later reader sees "attempted
6298            // again at T2" rather than silence.
6299            last.verified_head = Some(attempted_head);
6300            last.verified_at = Some(Timestamp::now());
6301            if verify_inconclusive(&last.e2e) {
6302                // Still not a real result: `e2e_deferred` is left exactly
6303                // as it was, so `needs_catchup_run` above reads
6304                // `ResourceBlocked` (via `e2e_status`, which checks
6305                // `resource_blocked` before `e2e_deferred`) and retries
6306                // again on the next reentry, rather than recording
6307                // contention as a red e2e and blocking the run on it.
6308                self.state.save()?;
6309                return Ok(());
6310            }
6311            last.e2e_deferred = false;
6312        }
6313        let last = &self.state.reviews[round_idx];
6314        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6315
6316        match last.e2e_status() {
6317            E2eStatus::Failed => {
6318                let red: Vec<String> = last
6319                    .e2e
6320                    .iter()
6321                    .filter(|o| !o.ok())
6322                    .map(|o| {
6323                        format!(
6324                            "`{}` -> {:?}\n{}",
6325                            o.command,
6326                            o.code,
6327                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6328                        )
6329                    })
6330                    .collect();
6331                self.state
6332                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6333                self.state.status = RunStatus::Blocked;
6334            }
6335            // `needs_catchup_run` above already retried once this call; if
6336            // it is still blocked, this is magi's own admission it could
6337            // not get a command to run, never a verdict on the patch — the
6338            // run is left exactly where a later reentry can retry again.
6339            E2eStatus::ResourceBlocked => {
6340                self.state.event(
6341                    "review",
6342                    format!(
6343                        "{why}; e2e could not run (shared build cache unavailable); not \
6344                         deciding yet"
6345                    ),
6346                );
6347            }
6348            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6349                self.state.event(
6350                    "review",
6351                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6352                );
6353                self.record_contested_handoff();
6354                self.state.status = RunStatus::Gating;
6355            }
6356        }
6357        self.state.save()?;
6358        Ok(())
6359    }
6360
6361    // ----------------------------------------------------------------- gate
6362
6363    async fn gate(&mut self) -> Result<()> {
6364        // Judged by the review record itself, not by `status`: a solo
6365        // candidate's `judge`/`deliberate` skip rewrites `status` on every
6366        // reentry (see `judge`), and trusting it here is exactly how a run
6367        // that exhausted its review budget got gated and merged a second
6368        // time around. `review_conclusion` recomputes the review loop's own
6369        // verdict from the round records themselves — `Gating` for a clean
6370        // round or a hand-off (see `stop_reviewing`), anything else means the
6371        // loop is still going or genuinely blocked.
6372        // A base the winner could not be replayed onto is a decision, not a
6373        // round: there is no landing tree to gate. Read as its own record for
6374        // the same reason the review verdict is.
6375        if self.state.status == RunStatus::Failed
6376            || self
6377                .state
6378                .base_sync
6379                .as_ref()
6380                .is_some_and(|s| s.conflict.is_some())
6381            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6382                != Some(RunStatus::Gating)
6383        {
6384            return Ok(());
6385        }
6386        if self.state.gate_ran {
6387            // `review_loop` derives its conclusion from the clean review
6388            // record on every reentry and therefore puts a completed run back
6389            // in `Gating`. A recorded gate is a stronger, terminal fact:
6390            // retain its original command output (or lack of any, for a repo
6391            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
6392            // doc) and restore `Blocked` on a real failure rather than
6393            // pretending the command is still running or running it a second
6394            // time. `gate_ran == false` remains the only shape — unattempted,
6395            // or a resource-blocked retry — that may still need to execute a
6396            // command.
6397            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6398                self.state.status = RunStatus::Blocked;
6399                self.state.save()?;
6400            }
6401            return Ok(());
6402        }
6403        let Some(winner) = self.state.winner().cloned() else {
6404            return Ok(());
6405        };
6406        self.state.status = RunStatus::Gating;
6407        let mut outcomes = self.run_gate(&winner).await?;
6408        loop {
6409            // A resource-blocked outcome means the gate command never actually
6410            // ran - the shared build cache could not be acquired or confirmed
6411            // fresh in time - which is evidence about the machine, not about
6412            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
6413            // Recording it as a red gate would mark a run `Blocked` on nothing
6414            // but contention magi has already logged; leaving `self.state.gate`
6415            // empty and `self.state.gate_ran` false instead keeps the shape
6416            // this function already treats as "still needs to run" (see the
6417            // early-return above), so the next call retries the command
6418            // rather than concluding anything.
6419            if verify_inconclusive(&outcomes) {
6420                self.state.save()?;
6421                return Ok(());
6422            }
6423            if outcomes.iter().all(CommandOutcome::ok) {
6424                break;
6425            }
6426            match self.gate_fix_round(&winner, &outcomes).await? {
6427                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6428                GateFix::Stop => break,
6429                GateFix::Defer => {
6430                    self.state.save()?;
6431                    return Ok(());
6432                }
6433            }
6434        }
6435        let passed = outcomes.iter().all(CommandOutcome::ok);
6436        self.state.gate = outcomes;
6437        self.state.gate_ran = true;
6438        self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6439        if !passed {
6440            self.state.status = RunStatus::Blocked;
6441            let spent = self.state.gate_fixes.len();
6442            self.state.event(
6443                "gate",
6444                if spent == 0 {
6445                    "gate failed; not merging".to_owned()
6446                } else {
6447                    format!("gate failed after {spent} gate-fix round(s); not merging")
6448                },
6449            );
6450        }
6451        self.state.save()?;
6452        Ok(())
6453    }
6454
6455    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
6456    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
6457    /// after review is clean and never on a candidate awaiting judging.
6458    ///
6459    /// Never fails the run: a non-zero exit or timeout is a warning and a
6460    /// recorded outcome, and the gate remains the single arbiter. Nothing
6461    /// configured means nothing happens - no event, no commit. `commit_all`
6462    /// commits any leftover change under the neutral identity and returns
6463    /// `false` when the tree is clean, so no empty commit is ever made.
6464    async fn run_pre_gate(&mut self, winner: &Candidate) {
6465        let commands = self.state.config.verify.pre_gate.clone();
6466        if commands.is_empty() {
6467            return;
6468        }
6469        let shell = self.state.config.shell();
6470        let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6471        let (outcomes, _) = run_commands(
6472            &mut self.state,
6473            "pre_gate",
6474            "pre_gate",
6475            0,
6476            &shell,
6477            &commands,
6478            &winner.worktree,
6479            timeout,
6480        )
6481        .await;
6482        for o in &outcomes {
6483            if !o.ok() {
6484                tracing::warn!(
6485                    "pre_gate `{}` failed ({:?}); the gate decides",
6486                    o.command,
6487                    o.code
6488                );
6489            }
6490            self.state.event(
6491                "pre_gate",
6492                format!(
6493                    "`{}` -> {}",
6494                    o.command,
6495                    if o.ok() {
6496                        "pass".to_owned()
6497                    } else {
6498                        format!(
6499                            "FAIL ({:?})\n{}",
6500                            o.code,
6501                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6502                        )
6503                    }
6504                ),
6505            );
6506        }
6507        self.state.pre_gate = outcomes;
6508        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
6509            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
6510                Ok(head) => {
6511                    self.state
6512                        .event("pre_gate", format!("committed mechanical fixes ({head})"));
6513                    self.state.pre_gate_commit = Some(head);
6514                }
6515                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
6516            },
6517            Ok(false) => {}
6518            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
6519        }
6520        if let Err(e) = self.state.save() {
6521            tracing::warn!("could not persist the pre_gate record: {e:#}");
6522        }
6523    }
6524
6525    /// Run `verify.gate` once against the winner's current tree, logging one
6526    /// event per command. Empty when nothing is configured.
6527    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
6528        self.run_pre_gate(winner).await;
6529        let shell = self.state.config.shell();
6530        let gate_commands = self.state.config.verify.gate.clone();
6531        // Zero commands has nothing to run and nothing that could touch the
6532        // shared build cache, so it never needs a lease: `Config::cache_dir`
6533        // is derived from `verify.e2e` too, so a repo with no `verify.gate`
6534        // commands but a `CARGO_TARGET_DIR`-using `verify.e2e` would
6535        // otherwise queue behind an unrelated run's lease and come back
6536        // resource-blocked - `gate_ran` would stay false on nothing but
6537        // cache contention, for a step that had nothing to check in the
6538        // first place.
6539        let outcomes = if gate_commands.is_empty() {
6540            Vec::new()
6541        } else {
6542            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6543            let cache_dir = self.state.config.cache_dir();
6544            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6545            let (outcomes, _) = with_cache_lease(
6546                &mut self.state,
6547                cache_dir.as_deref(),
6548                "gate",
6549                "gate",
6550                &winner.worktree,
6551                &head,
6552                timeout,
6553                "final gate",
6554                |state, budget| {
6555                    let shell = shell.clone();
6556                    let gate_commands = gate_commands.clone();
6557                    let worktree = winner.worktree.clone();
6558                    async move {
6559                        let (outcomes, timed_out_pids) = run_commands(
6560                            state,
6561                            "gate",
6562                            "gate",
6563                            0,
6564                            &shell,
6565                            &gate_commands,
6566                            &worktree,
6567                            budget,
6568                        )
6569                        .await;
6570                        (outcomes, false, timed_out_pids)
6571                    }
6572                },
6573            )
6574            .await;
6575            outcomes
6576        };
6577        if outcomes.is_empty() {
6578            // Nothing configured to check — distinct from every other
6579            // silence in this run's event log, since an empty `gate` alone
6580            // no longer says whether the gate ran at all (see
6581            // `RunState::gate_ran`'s own doc).
6582            self.state.event(
6583                "gate",
6584                "no gate commands configured; nothing to check, passing",
6585            );
6586        }
6587        for o in &outcomes {
6588            self.state.event(
6589                "gate",
6590                format!(
6591                    "`{}` -> {}",
6592                    o.command,
6593                    if o.ok() {
6594                        "pass".to_owned()
6595                    } else {
6596                        format!(
6597                            "FAIL ({:?})\n{}",
6598                            o.code,
6599                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6600                        )
6601                    }
6602                ),
6603            );
6604        }
6605        Ok(outcomes)
6606    }
6607
6608    /// One bounded fix round for a failing gate.
6609    ///
6610    /// The fixer is told the failure came from the gate itself, not from a
6611    /// reviewer, and is shown the failed commands, their exit codes and a tail
6612    /// of their output - whatever `[verify].gate` holds, nothing here knows
6613    /// what those commands run. Only a normal non-zero exit that printed
6614    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
6615    /// command or a full disk says nothing about the code, and a fixer sent
6616    /// after it can only appease the machine. The round is judged by what git
6617    /// says moved, never by the fixer's own report, and `verify.e2e` runs
6618    /// again before the gate does, so a fix cannot trade a green gate for a
6619    /// red e2e unnoticed.
6620    async fn gate_fix_round(
6621        &mut self,
6622        winner: &Candidate,
6623        outcomes: &[CommandOutcome],
6624    ) -> Result<GateFix> {
6625        let cap = self.state.config.graph.gate_fix_rounds;
6626        let spent = self.state.gate_fixes.len();
6627        if spent >= cap {
6628            if cap > 0 {
6629                self.state.event(
6630                    "gate",
6631                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
6632                );
6633            }
6634            return Ok(GateFix::Stop);
6635        }
6636        if !gate_fixable(outcomes) {
6637            self.state.event(
6638                "gate",
6639                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
6640                 command or similar); not spending a fix round on it",
6641            );
6642            return Ok(GateFix::Stop);
6643        }
6644        let min_free = self.state.config.disk.min_free_bytes;
6645        if min_free > 0 {
6646            match crate::disk::free_bytes(&winner.worktree) {
6647                Ok(free) if crate::disk::enough_space(free, min_free) => {}
6648                Ok(free) => {
6649                    self.state.event(
6650                        "gate",
6651                        format!(
6652                            "only {free} bytes free ({min_free} required by `[disk] \
6653                             min_free_bytes`); not spending a fix round on a failure the disk \
6654                             may explain"
6655                        ),
6656                    );
6657                    return Ok(GateFix::Stop);
6658                }
6659                Err(e) => {
6660                    self.state.event(
6661                        "gate",
6662                        format!("free disk space could not be measured ({e:#}); no fix round"),
6663                    );
6664                    return Ok(GateFix::Stop);
6665                }
6666            }
6667        }
6668
6669        let attempt = spent + 1;
6670        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
6671        let base = self.landing_base();
6672        let fix_prompt = prompt::gate_fix(
6673            &self.state.instruction,
6674            &failed,
6675            attempt,
6676            cap,
6677            &self.state.config.graph.language,
6678        );
6679        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6680        let sessions = self.state.config.graph.sessions;
6681        let artifacts = agent::artifacts_dir(&self.state.dir());
6682        self.state.event(
6683            "gate",
6684            format!("gate failed; gate-fix round {attempt} of {cap}"),
6685        );
6686        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6687        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6688        let (_, seat, out) = self
6689            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
6690                prompt: fix_prompt.clone(),
6691                spec,
6692                seat,
6693                cwd: winner.worktree.clone(),
6694                timeout,
6695                allow_write: true,
6696                sessions,
6697                artifacts: artifacts.clone(),
6698                stem: format!("gate-fix-{attempt}"),
6699                handover: None,
6700            })
6701            .await;
6702        // The fixer was never asked: no round is spent (nothing is recorded)
6703        // and a later reentry asks the same agent again.
6704        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6705            return Ok(GateFix::Defer);
6706        }
6707        let mut record = GateFixRecord {
6708            agent: seat.agent.clone(),
6709            failed,
6710            notes: String::new(),
6711            committed: false,
6712            error: None,
6713        };
6714        match out {
6715            AgentOutcome::Ok(o) => {
6716                // A missing report is not a failed fix: the round is judged
6717                // by the tree below, and the report only carries prose.
6718                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
6719                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6720                }
6721            }
6722            AgentOutcome::Dropped(_) => {
6723                record.error = Some("the CLI dropped the stream".to_owned());
6724            }
6725            AgentOutcome::Quota(o) => {
6726                self.state.quota.push(QuotaLoss {
6727                    seat: seat.key.clone(),
6728                    node: "gate-fix".to_owned(),
6729                    at: Timestamp::now(),
6730                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6731                });
6732                record.error = Some("rate limited (quota); fixer could not run".to_owned());
6733            }
6734            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
6735        }
6736        self.state.seats.insert(seat.key.clone(), seat);
6737        if let Ok(r) = git::rescue_commit(
6738            &winner.worktree,
6739            &format!("magi: gate fix {attempt} (uncommitted work)"),
6740        )
6741        .await
6742        {
6743            self.state.note_withheld("gate-fix", &r.withheld);
6744        }
6745        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6746        record.committed = after != before;
6747        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
6748        let note = record.error.clone();
6749        self.state.gate_fixes.push(record);
6750        self.state.save()?;
6751        if !changed {
6752            self.state.event(
6753                "gate",
6754                match note {
6755                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
6756                    None => format!("gate-fix round {attempt}: the tree did not change"),
6757                },
6758            );
6759            return Ok(GateFix::Stop);
6760        }
6761        self.state.event(
6762            "gate",
6763            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
6764        );
6765
6766        let commands = self.state.config.verify.e2e.clone();
6767        if !commands.is_empty() {
6768            let shell = self.state.config.shell();
6769            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6770            let cache_dir = self.state.config.cache_dir();
6771            let context = format!("gate-fix round {attempt}");
6772            let (e2e, _) = with_cache_lease(
6773                &mut self.state,
6774                cache_dir.as_deref(),
6775                "e2e",
6776                "e2e",
6777                &winner.worktree,
6778                &after,
6779                timeout,
6780                &context,
6781                |state, budget| {
6782                    let shell = shell.clone();
6783                    let commands = commands.clone();
6784                    let context = context.clone();
6785                    let worktree = winner.worktree.clone();
6786                    async move {
6787                        run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
6788                            .await
6789                    }
6790                },
6791            )
6792            .await;
6793            if verify_inconclusive(&e2e) {
6794                return Ok(GateFix::Defer);
6795            }
6796            if e2e.iter().any(|o| !o.ok()) {
6797                self.state.event(
6798                    "gate",
6799                    format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
6800                );
6801                return Ok(GateFix::Stop);
6802            }
6803        }
6804        Ok(GateFix::Retry)
6805    }
6806
6807    // ---------------------------------------------------------------- merge
6808
6809    /// One read-only rewrite by the summary's author; a text the gate still
6810    /// withholds goes to the owner (see `github_text`) and the fixed fallback
6811    /// applies on their word or on silence. `None` means the run is parked on
6812    /// that question and nothing may be pushed or opened yet.
6813    async fn guarded_pr_message(
6814        &mut self,
6815        winner: &Candidate,
6816        facts: Option<&BranchFacts>,
6817        posting: bool,
6818    ) -> Result<Option<PrMessage>> {
6819        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6820        let head = match final_head.as_deref() {
6821            Some(h) => crate::pr_outcome::FinalHead::At(h),
6822            None => crate::pr_outcome::FinalHead::Unknown,
6823        };
6824        let mut pr = pr_message_raw(&self.state, winner.label, facts, head);
6825        if !posting {
6826            let identity = crate::scrub::Identity::current();
6827            return Ok(Some(PrMessage {
6828                title: crate::scrub::scrub(&pr.title, &identity),
6829                body: crate::scrub::scrub(&pr.body, &identity),
6830            }));
6831        }
6832        // This very text was already rejected and asked about: no second
6833        // judge call, rewrite or question, just carry the record on.
6834        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
6835        if let Some(known) = self
6836            .state
6837            .github_text
6838            .clone()
6839            .filter(|g| g.fingerprint == raw_fp)
6840        {
6841            return self.settle_withheld(winner, &pr, known).await;
6842        }
6843        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
6844        let mut violations = crate::github_text::check_with(&pr.title, &pr.body, decision);
6845        let identity = crate::scrub::Identity::current();
6846        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
6847            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
6848            && !violations.contains(&crate::github_text::Violation::SensitiveData)
6849        {
6850            violations.push(crate::github_text::Violation::SensitiveData);
6851        }
6852        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
6853        // only a language violation starts one; sensitive data alone goes
6854        // straight to the owner, with the redacted text as the fallback.
6855        let has_language = violations
6856            .iter()
6857            .any(|v| *v != crate::github_text::Violation::SensitiveData);
6858        if self.state.config.graph.github_text_guard && !violations.is_empty() {
6859            self.state.event(
6860                "github-text",
6861                if has_language {
6862                    format!("description rejected: {violations:?}; requesting one rewrite")
6863                } else {
6864                    format!("description rejected: {violations:?}; asking the owner")
6865                },
6866            );
6867            let mut rewritten = false;
6868            let mut rejected_rewrite = None;
6869            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
6870            {
6871                let output = text;
6872                if let Some(title) = summary_title(&output) {
6873                    // Judge the redacted candidate that will actually be posted.
6874                    // Sensitive data alone must not discard a useful rewrite.
6875                    let raw_body = summary_without_title(&output);
6876                    let clean_title = crate::scrub::scrub(&title, &identity);
6877                    let mut body = crate::scrub::scrub(&raw_body, &identity);
6878                    let redacted = clean_title != title || body != raw_body;
6879                    let title = clean_title;
6880                    let marker = format!("magi:run/{}", self.state.id);
6881                    if !body.trim().is_empty() && !body.contains(&marker) {
6882                        body.push_str(&format!("\n\n{marker}"));
6883                    }
6884                    let rewrite = PrMessage {
6885                        title: title.clone(),
6886                        body: body.clone(),
6887                    };
6888                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
6889                    let mut failed = crate::github_text::check_with(&title, &body, redecision);
6890                    if (!crate::github_text::shareable(&title)
6891                        || !crate::github_text::shareable(&body))
6892                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
6893                    {
6894                        failed.push(crate::github_text::Violation::SensitiveData);
6895                    }
6896                    if !title.trim().is_empty() && !body.trim().is_empty() && !failed.is_empty() {
6897                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
6898                            title: title.clone(),
6899                            body: body.clone(),
6900                            failed: failed
6901                                .iter()
6902                                .map(|v| crate::github_text::category(*v).to_owned())
6903                                .collect(),
6904                        });
6905                    }
6906                    if !title.trim().is_empty() && !body.trim().is_empty() && failed.is_empty() {
6907                        decision = redecision;
6908                        pr = PrMessage { title, body };
6909                        rewritten = true;
6910                        self.state
6911                            .event("github-text", "description rewrite passed");
6912                        if redacted {
6913                            self.state.event(
6914                                "github-text",
6915                                "sensitive data removed from description rewrite",
6916                            );
6917                        }
6918                    }
6919                }
6920            }
6921            if !rewritten {
6922                self.state.event(
6923                    "github-text",
6924                    "description rewrite unavailable or rejected; asking the owner",
6925                );
6926                if let Some(w) =
6927                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
6928                {
6929                    // A different text than the one asked about before: that
6930                    // question no longer describes anything.
6931                    self.retire_github_text_question("the withheld text changed");
6932                    let record = crate::run::GithubTextAsk {
6933                        rewrite: rejected_rewrite,
6934                        fingerprint: raw_fp,
6935                        title: w.title,
6936                        body: w.body,
6937                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
6938                        question: None,
6939                        asks: 0,
6940                        resolved: false,
6941                        chosen_title: None,
6942                        choice: None,
6943                    };
6944                    return self.settle_withheld(winner, &pr, record).await;
6945                }
6946            }
6947        }
6948        let (title, body) =
6949            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
6950        Ok(Some(PrMessage { title, body }))
6951    }
6952
6953    /// One read-only rewrite request for a withheld description. The
6954    /// implementer's own conversation is reused only when it exists and can be
6955    /// resumed; otherwise the `[roles] language_judge` chain (or the default
6956    /// pick) writes it in a separate conversation. The prompt carries the whole
6957    /// text, so no agent needs any memory. Each agent is asked at most once and
6958    /// the chain advances on an error, quota, unusable answer or a reply with
6959    /// no `TITLE:` line. Nothing is recorded as a handover.
6960    async fn rewrite_pr_text(
6961        &mut self,
6962        winner: &Candidate,
6963        pr: &PrMessage,
6964        violations: &[crate::github_text::Violation],
6965    ) -> Option<String> {
6966        let prompt = format!(
6967            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the complete Markdown body. Preserve the magi run marker.\n\nTITLE: {}\n{}",
6968            pr.title, pr.body
6969        );
6970        let sessions = self.state.config.graph.sessions;
6971        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
6972        // (spec, seat, resumed)
6973        let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
6974        let key = format!("impl-{}", winner.label);
6975        if let Some(spec) = self
6976            .state
6977            .config
6978            .agents
6979            .iter()
6980            .find(|a| a.id == winner.agent)
6981            .cloned()
6982            && let Some(existing) = self.state.seats.get(&key)
6983            && existing.agent == spec.id
6984            && agent::has_session(spec.kind, existing, sessions)
6985        {
6986            attempts.push((spec, existing.clone(), true));
6987        } else {
6988            let chain = agent::pick_chain(
6989                &self.state.config.agents,
6990                self.state.config.roles.language_judge.as_ref(),
6991                &agent::installed,
6992                "language_judge",
6993            );
6994            match chain {
6995                Ok(chain) => {
6996                    for spec in chain {
6997                        let seat = crate::fixer::seat_for(
6998                            &mut self.state,
6999                            "github-text-rewrite",
7000                            &spec.id,
7001                        );
7002                        attempts.push((spec, seat, false));
7003                    }
7004                }
7005                Err(e) => {
7006                    tracing::warn!("no agent for the description rewrite: {e:#}");
7007                    self.state
7008                        .event("github-text", "no rewrite agent available");
7009                    return None;
7010                }
7011            }
7012        }
7013        let prompts = self.state.config.prompts.clone();
7014        let cache = self.state.config.cache_dir();
7015        let run = self.state.id.clone();
7016        for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7017            let job = SeatJob {
7018                spec: spec.clone(),
7019                seat,
7020                cwd: winner.worktree.clone(),
7021                prompt: prompt.clone(),
7022                timeout: retry_budget(timeout_full, resumed),
7023                allow_write: false,
7024                sessions,
7025                artifacts: agent::artifacts_dir(&self.state.dir()),
7026                stem: if i == 0 {
7027                    "github-text-rewrite".to_owned()
7028                } else {
7029                    format!("github-text-rewrite-{}", spec.id)
7030                },
7031                handover: None,
7032            };
7033            let ctx = WaveCtx {
7034                run: &run,
7035                node: "github-text",
7036                prompts: &prompts,
7037                cache: cache.as_deref(),
7038                round: None,
7039                carry_seats: false,
7040                pauses: Some((&self.pause, &self.interrupt)),
7041            };
7042            let (seat, outcome) =
7043                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7044            self.state.seats.insert(seat.key.clone(), seat);
7045            match outcome {
7046                AgentOutcome::Ok(o)
7047                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7048                {
7049                    return Some(o.text);
7050                }
7051                AgentOutcome::ResourceBlocked(_) => return None,
7052                _ => {}
7053            }
7054        }
7055        None
7056    }
7057
7058    /// Abandon the standing posting-gate question, if any.
7059    fn retire_github_text_question(&mut self, why: &str) {
7060        let Some(id) = self
7061            .state
7062            .github_text
7063            .as_ref()
7064            .and_then(|g| g.question.clone())
7065        else {
7066            return;
7067        };
7068        let _ = ask::Questions::open().update(&id, |q| {
7069            q.abandon(why);
7070            Ok(())
7071        });
7072    }
7073
7074    /// Carry a withheld text to its end: file the question, park while it is
7075    /// open, and on an answer, silence or timeout pick the text to post. The
7076    /// decision is saved before it is used, so it applies once.
7077    async fn settle_withheld(
7078        &mut self,
7079        winner: &Candidate,
7080        pr: &PrMessage,
7081        mut g: crate::run::GithubTextAsk,
7082    ) -> Result<Option<PrMessage>> {
7083        use crate::github_text as gt;
7084        let store = ask::Questions::open();
7085        let timeout = self.state.config.graph.answer_timeout;
7086        if !g.resolved {
7087            let mut standing = g
7088                .question
7089                .as_ref()
7090                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7091            if let Some(q) = standing.as_ref()
7092                && q.status.open()
7093                && gt::expired(q, timeout)
7094            {
7095                let why = format!("no answer within {}s of asking", timeout.max(1));
7096                standing = store
7097                    .update(&q.id, |q| {
7098                        q.abandon(&why);
7099                        Ok(())
7100                    })
7101                    .ok()
7102                    .map(|(q, ())| q);
7103            }
7104            let mut ask_again: Option<Option<Vec<String>>> = None;
7105            match standing {
7106                // Recorded but never written (a stop between the two saves),
7107                // or removed: ask again without spending the retry.
7108                None if g.asks <= 1 => {
7109                    g.asks = 0;
7110                    ask_again = Some(None);
7111                }
7112                None => {
7113                    g.resolved = true;
7114                    g.chosen_title = None;
7115                    g.choice = Some(GithubTextChoice::Fallback);
7116                }
7117                Some(q) if q.status.open() => {
7118                    self.park_on_github_text(&g)?;
7119                    return Ok(None);
7120                }
7121                Some(q) => match gt::read_reply(&q) {
7122                    gt::Reply::Fallback => {
7123                        g.resolved = true;
7124                        g.chosen_title = None;
7125                        g.choice = Some(GithubTextChoice::Fallback);
7126                    }
7127                    gt::Reply::Original => {
7128                        g.resolved = true;
7129                        g.chosen_title = None;
7130                        g.choice = Some(GithubTextChoice::Original);
7131                    }
7132                    gt::Reply::Rewrite => {
7133                        g.resolved = true;
7134                        g.chosen_title = None;
7135                        g.choice = Some(if g.rewrite.is_some() {
7136                            GithubTextChoice::Rewrite
7137                        } else {
7138                            GithubTextChoice::Fallback
7139                        });
7140                    }
7141                    gt::Reply::Title(text) => {
7142                        let first = text
7143                            .lines()
7144                            .map(str::trim)
7145                            .find(|l| !l.is_empty())
7146                            .unwrap_or("");
7147                        let decision = if first.is_empty() {
7148                            None
7149                        } else {
7150                            let candidate = PrMessage {
7151                                title: first.to_owned(),
7152                                body: String::new(),
7153                            };
7154                            self.judge_pr_language(&winner.worktree, &candidate).await
7155                        };
7156                        match gt::vet_title(&text, decision) {
7157                            Ok(title) => {
7158                                g.resolved = true;
7159                                g.chosen_title = Some(title);
7160                                g.choice = Some(GithubTextChoice::MyTitle);
7161                            }
7162                            Err(cats) if g.asks < 2 => {
7163                                ask_again = Some(Some(
7164                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7165                                ))
7166                            }
7167                            Err(_) => {
7168                                g.resolved = true;
7169                                g.chosen_title = None;
7170                                g.choice = Some(GithubTextChoice::Fallback);
7171                            }
7172                        }
7173                    }
7174                },
7175            }
7176            if let Some(rejected) = ask_again {
7177                // A retry (after a rejected replacement title) is still about
7178                // the original text: the same ORIGINAL / FALLBACK comparison.
7179                let w = gt::Withheld {
7180                    title: g.title,
7181                    body: g.body,
7182                    categories: g.categories.clone(),
7183                };
7184                // Named by the original text's fingerprint alone, so a resume
7185                // rewrites the same file and the detail stays identical.
7186                let artifact = match crate::run::write_artifact(
7187                    &self.state,
7188                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
7189                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7190                ) {
7191                    Ok(p) => Some(p),
7192                    Err(e) => {
7193                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
7194                        None
7195                    }
7196                };
7197                let mut q = ask::Question::new(
7198                    self.state.id.clone(),
7199                    gt::ASK_NODE.to_owned(),
7200                    gt::ASK_SEAT.to_owned(),
7201                    gt::question_summary(&self.state.config.graph.language, &w),
7202                    gt::question_detail(
7203                        &w,
7204                        &pr.title,
7205                        &pr.body,
7206                        &self.state.id,
7207                        rejected.as_deref(),
7208                        artifact.as_deref(),
7209                        g.rewrite.as_ref(),
7210                    ),
7211                    gt::question_choices(&w, g.rewrite.as_ref()),
7212                );
7213                q.answer_timeout = timeout;
7214                // Recorded before it is written: a stop in between leaves a
7215                // record naming a question the resume then files again.
7216                g.question = Some(q.id.clone());
7217                g.asks += 1;
7218                self.state.github_text = Some(g.clone());
7219                self.state.event(
7220                    "github-text",
7221                    format!("asking the owner about the withheld text ({})", q.short()),
7222                );
7223                self.state.save()?;
7224                store
7225                    .put(&mut q)
7226                    .context("file the posting-gate question")?;
7227                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7228                    tracing::warn!(
7229                        "could not notify about posting-gate question {}: {e:#}",
7230                        q.short()
7231                    );
7232                }
7233                self.park_on_github_text(&g)?;
7234                return Ok(None);
7235            }
7236        }
7237        let resolved = g.chosen_title.clone();
7238        let original = g.choice == Some(GithubTextChoice::Original);
7239        // Field names only: the event log never records the text.
7240        self.state.event(
7241            "github-text",
7242            if original {
7243                format!(
7244                    "posting the original text for: {}",
7245                    match (g.title, g.body) {
7246                        (true, true) => "title, description",
7247                        (true, false) => "title",
7248                        _ => "description",
7249                    }
7250                )
7251            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7252                "posting the saved rewrite title and description".to_owned()
7253            } else if resolved.is_some() {
7254                "posting the owner's replacement title".to_owned()
7255            } else {
7256                "posting the neutral text for the withheld fields".to_owned()
7257            },
7258        );
7259        self.state.github_text = Some(g.clone());
7260        self.state.save()?;
7261        if g.choice == Some(GithubTextChoice::Rewrite)
7262            && let Some(r) = g.rewrite.as_ref()
7263        {
7264            return Ok(Some(PrMessage {
7265                title: r.title.clone(),
7266                body: r.body.clone(),
7267            }));
7268        }
7269        let w = gt::Withheld {
7270            title: g.title,
7271            body: g.body,
7272            categories: g.categories.clone(),
7273        };
7274        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
7275        if let Some(t) = resolved {
7276            title = t;
7277        }
7278        // Every withheld field is now neutral or vetted English.
7279        let english = gt::LanguageDecision {
7280            title_english: true,
7281            body_english: true,
7282            body_complete: true,
7283        };
7284        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
7285        if original {
7286            // The owner's explicit choice overrides the gate for the fields it
7287            // was made for; the others keep what the gate produced.
7288            if g.title {
7289                title = pr.title.clone();
7290            }
7291            if g.body {
7292                body = pr.body.clone();
7293            }
7294        }
7295        Ok(Some(PrMessage { title, body }))
7296    }
7297
7298    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
7299        self.state.github_text = Some(g.clone());
7300        self.state.end_pass_parked(ParkCause::Question);
7301        self.state.parked = true;
7302        self.state.event(
7303            "github-text",
7304            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
7305        );
7306        self.state.save()
7307    }
7308
7309    /// Ask `[roles] language_judge` about this text and record which source
7310    /// decided; the text itself is never recorded.
7311    async fn judge_pr_language(
7312        &mut self,
7313        cwd: &Path,
7314        pr: &PrMessage,
7315    ) -> Option<crate::github_text::LanguageDecision> {
7316        self.state.config.roles.language_judge.as_ref()?;
7317        let decision =
7318            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
7319        self.state.event(
7320            "github-text",
7321            if decision.is_some() {
7322                "language decided by the language judge"
7323            } else {
7324                "language judge unavailable; using built-in heuristics"
7325            },
7326        );
7327        decision
7328    }
7329
7330    async fn merge(&mut self) -> Result<()> {
7331        // Same reasoning as `gate`: ask the review and gate records directly
7332        // rather than `status`, which a solo-candidate `judge`/`deliberate`
7333        // skip can rewrite on reentry to something that no longer says
7334        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
7335        // so a hand-off (open findings, green verification) reaches merge
7336        // exactly like a genuinely clean round does.
7337        //
7338        // A run resumed mid-`land` never reaches here at all: `execute`
7339        // recognises `RunStatus::Landing` before it even calls `prep`, and
7340        // routes straight to `run_land` instead. That has to happen a level
7341        // up from this function, not with a check in here, because
7342        // `review_loop`'s own status recomputation (see its doc) runs
7343        // *before* `merge` on every reentry and would otherwise overwrite
7344        // the `Landing` marker with `Gating` before this node ever saw it.
7345        if self
7346            .state
7347            .base_sync
7348            .as_ref()
7349            .is_some_and(|s| s.conflict.is_some())
7350            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7351                != Some(RunStatus::Gating)
7352            // `gate_ran == false` is not "passed" - `gate` leaves it false
7353            // both before it has ever run and when its last attempt was
7354            // resource-blocked (see `Runner::gate`'s own doc), and neither is
7355            // permission to merge on nothing but the review record. Only a
7356            // gate that actually ran - zero commands configured and
7357            // vacuously passed, or one or more that all exited 0 - may
7358            // proceed; `RunState::gate_status` is the single place that
7359            // reading is computed.
7360            || !self.state.gate_status().ok()
7361        {
7362            return Ok(());
7363        }
7364        // This node's own record, not `status`: `status == Ready` is not
7365        // unique to the harmless `MergeMode::None` path this line was
7366        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
7367        // run's PR was closed without merging — and on that run `mode` is
7368        // still `Pr`, so a reentry that fell through here would push and
7369        // open a second pull request. `self.state.merge` is set exactly once
7370        // this node (or `land`) has already produced a verdict, under every
7371        // mode, which is what "already done" actually means here.
7372        if self.state.merge.is_some() && !self.state.pr_publication_pending {
7373            return Ok(());
7374        }
7375        let Some(winner) = self.state.winner().cloned() else {
7376            return Ok(());
7377        };
7378        let repo = self.state.repo.clone();
7379        let base = self.state.base_branch.clone();
7380        let mode = self.state.config.merge.mode;
7381        let style = self.state.config.merge.style;
7382        let facts = if is_review_run(&self.state) {
7383            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
7384            let start = review_base(
7385                &repo,
7386                &self.state.config.merge.remote,
7387                &base,
7388                &self.state.base_commit,
7389                &winner.branch,
7390            )
7391            .await;
7392            branch_facts(&repo, &start, &winner.branch).await
7393        } else {
7394            None
7395        };
7396        // `None` when the base could not be freshly read: then an adopted
7397        // pull request's title is left alone.
7398        let leaked = if is_review_run(&self.state) {
7399            leaked_subjects(&self.state, &winner.branch).await
7400        } else {
7401            Some(Vec::new())
7402        };
7403        // `None`: parked on the owner's word about a withheld title or body.
7404        // Nothing has been pushed or opened, and `merge` stays unrecorded so
7405        // the resume arrives here again.
7406        let Some(pr) = self
7407            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
7408            .await?
7409        else {
7410            return Ok(());
7411        };
7412        let message = pr.commit_message();
7413
7414        let outcome = match mode {
7415            MergeMode::None => MergeOutcome {
7416                mode,
7417                ok: true,
7418                detail: manual_merge_command(style, &repo, &winner.branch, &message),
7419                empty: false,
7420            },
7421            MergeMode::Pr | MergeMode::Local
7422                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
7423            {
7424                MergeOutcome {
7425                    mode,
7426                    ok: false,
7427                    detail: empty_candidate_detail(&self.state, &base),
7428                    empty: true,
7429                }
7430            }
7431            MergeMode::Local => {
7432                let on = git::current_branch(&repo).await?;
7433                if on.as_deref() != Some(base.as_str()) {
7434                    MergeOutcome {
7435                        mode,
7436                        ok: false,
7437                        detail: format!(
7438                            "{} has {} checked out, not the base branch {base}",
7439                            repo.display(),
7440                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
7441                        ),
7442                        empty: false,
7443                    }
7444                } else if !git::is_clean(&repo).await? {
7445                    MergeOutcome {
7446                        mode,
7447                        ok: false,
7448                        detail: format!("{} is dirty; refusing to merge", repo.display()),
7449                        empty: false,
7450                    }
7451                } else {
7452                    let out = match style {
7453                        MergeStyle::Merge => {
7454                            git::merge_no_ff(&repo, &winner.branch, &message).await?
7455                        }
7456                        MergeStyle::Squash => {
7457                            git::merge_squash(&repo, &winner.branch, &message).await?
7458                        }
7459                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
7460                    };
7461                    MergeOutcome {
7462                        mode,
7463                        ok: out.ok(),
7464                        detail: if out.ok() { out.stdout } else { out.stderr },
7465                        empty: false,
7466                    }
7467                }
7468            }
7469            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
7470                self.state.pr_publication_pending = true;
7471                self.state.save()?;
7472                let out =
7473                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
7474                self.state.pr_publication_pending = out.is_err();
7475                MergeOutcome {
7476                    mode,
7477                    ok: out.is_ok(),
7478                    detail: match out {
7479                        Ok(url) => url,
7480                        Err(e) => format!("{e:#}"),
7481                    },
7482                    empty: false,
7483                }
7484            }
7485            MergeMode::Pr => {
7486                let remote = self.state.config.merge.remote.clone();
7487                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
7488                if !pushed.ok() {
7489                    MergeOutcome {
7490                        mode,
7491                        ok: false,
7492                        detail: pushed.stderr,
7493                        empty: false,
7494                    }
7495                } else {
7496                    // A retry or resume of a run whose branch already has an
7497                    // open pull request adopts it rather than failing on a
7498                    // duplicate. Only this winner branch into this base:
7499                    // `branch_for` derives the name from the run id, so a
7500                    // different run's pull request never matches.
7501                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
7502                    let out = match pr_merge_plan(found) {
7503                        PrPlan::Create => {
7504                            gh_pr_create(
7505                                &winner.worktree,
7506                                &base,
7507                                &winner.branch,
7508                                &pr.title,
7509                                &pr.body,
7510                            )
7511                            .await
7512                        }
7513                        PrPlan::Adopt { url, title } => {
7514                            self.state
7515                                .event("merge", format!("Pr: adopted open pull request {url}"));
7516                            if self.state.github_text.as_ref().is_some_and(|g| {
7517                                g.resolved
7518                                    && g.choice == Some(GithubTextChoice::Rewrite)
7519                                    && g.rewrite.is_some()
7520                            }) {
7521                                // The owner chose the entire pair, even if the
7522                                // existing title already matches or was edited.
7523                                land::set_pr_text_verbatim(
7524                                    &winner.worktree,
7525                                    &url,
7526                                    &pr.title,
7527                                    &pr.body,
7528                                )
7529                                .await
7530                                .map(|()| url)
7531                            } else {
7532                                if title != pr.title
7533                                    && (!is_review_run(&self.state)
7534                                        || leaked
7535                                            .as_deref()
7536                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
7537                                    && let Err(e) =
7538                                        if self.state.github_text.as_ref().is_some_and(|g| {
7539                                            g.resolved
7540                                                && g.title
7541                                                && g.choice == Some(GithubTextChoice::Original)
7542                                        }) {
7543                                            // The owner chose to post this title unmodified.
7544                                            land::set_pr_title_verbatim(
7545                                                &winner.worktree,
7546                                                &url,
7547                                                &pr.title,
7548                                            )
7549                                            .await
7550                                        } else {
7551                                            land::set_pr_title(
7552                                                &mut self.state,
7553                                                &winner.worktree,
7554                                                &url,
7555                                                &pr.title,
7556                                            )
7557                                            .await
7558                                        }
7559                                {
7560                                    tracing::warn!("could not refresh title of {url}: {e:#}");
7561                                    self.state
7562                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
7563                                }
7564                                Ok(url)
7565                            }
7566                        }
7567                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
7568                    };
7569                    match out {
7570                        Ok(url) => MergeOutcome {
7571                            mode,
7572                            ok: true,
7573                            detail: url,
7574                            empty: false,
7575                        },
7576                        Err(e) => MergeOutcome {
7577                            mode,
7578                            ok: false,
7579                            detail: e.to_string(),
7580                            empty: false,
7581                        },
7582                    }
7583                }
7584            }
7585        };
7586
7587        self.state.status = match (mode, outcome.ok) {
7588            (MergeMode::None, _) => RunStatus::Ready,
7589            (_, true) => RunStatus::Merged,
7590            (_, false) => RunStatus::Blocked,
7591        };
7592        self.state.event(
7593            "merge",
7594            format!(
7595                "{:?}: {}",
7596                mode,
7597                outcome.detail.lines().next().unwrap_or("")
7598            ),
7599        );
7600        self.state.merge = Some(outcome);
7601        self.state.save()?;
7602
7603        // The PR is open and the run would historically stop here, leaving the
7604        // operator to watch checks, feed review comments back to a fixer, and
7605        // merge. That was done by hand six times in one session before this
7606        // existed. Opt-in, because merging is the one irreversible thing magi
7607        // can do to a repository.
7608        if self.state.config.graph.land
7609            && mode == MergeMode::Pr
7610            && self.state.status == RunStatus::Merged
7611        {
7612            self.run_land().await?;
7613        }
7614        // `run_land` may have left `status` at `Landing` - still waiting on
7615        // CI or the owner's approval, not actually settled - so this has to
7616        // read whatever `status` ended up as here, not the `Merged` this
7617        // function set a few lines up.
7618        self.settle_questions();
7619        Ok(())
7620    }
7621
7622    /// Enter `land`.
7623    ///
7624    /// Shared between a fresh run's first pass through [`Runner::merge`] and
7625    /// a resumed run's re-entry. `land::land` itself is what serialises the
7626    /// two git-mutating moments inside the loop — the rebase push and
7627    /// `gh pr merge` — per repository (see its own doc); nothing here needs
7628    /// to hold a lock across the whole call, and doing so would serialise
7629    /// this run's CI wait against a *different* run's land-approval resume
7630    /// in the same repository, which is exactly the "must not wait on
7631    /// another task" property the daemon's slot-freeing exists to give.
7632    async fn run_land(&mut self) -> Result<()> {
7633        crate::pr_head::ensure_owner(&self.state)?;
7634        let url = self
7635            .state
7636            .merge
7637            .as_ref()
7638            .map(|m| m.detail.clone())
7639            .unwrap_or_default();
7640        let url = url.lines().next().unwrap_or("").trim().to_owned();
7641        if !url.starts_with("http") {
7642            return Ok(());
7643        }
7644        // A land failure is not a lost run: the work is on a branch and the
7645        // pull request is open, which is exactly where a human takes over.
7646        match land::land(&mut self.state, &url).await {
7647            Ok(pr) if self.state.parked => {
7648                // `land` already saved the parked marker; nothing here
7649                // overrides `status` back to a terminal value while an
7650                // approval is still outstanding.
7651                let _ = pr;
7652            }
7653            Ok(pr) => {
7654                self.state.status = match pr.state {
7655                    land::PrLifecycle::Merged => RunStatus::Merged,
7656                    _ => RunStatus::Blocked,
7657                };
7658                // Downstream of a confirmed merge only - see
7659                // `bump::should_release_bump`'s own doc for why this one
7660                // check covers all three of `land`'s success paths.
7661                // Best-effort: the run already landed, so a failure here
7662                // (the decision call, `gh`, `cargo`) is recorded and never
7663                // turns a landed run into a failed one.
7664                if bump::should_release_bump(self.state.status)
7665                    && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
7666                {
7667                    // The event is the run's own record. Not-eligible cases
7668                    // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
7669                    // `Err` is a bump that was tried and failed:
7670                    // `after_merge` itself raises the operator notice for
7671                    // that, whether or not a release PR exists yet.
7672                    self.state
7673                        .event("bump", format!("release bump skipped: {e:#}"));
7674                }
7675                // Independent of the bump, and best-effort in the same way:
7676                // findings the merge left open become follow-up tasks.
7677                if self.state.status == RunStatus::Merged {
7678                    crate::followup::after_merge(&mut self.state, &pr.url).await;
7679                }
7680                self.state.save()?;
7681            }
7682            Err(e) => {
7683                self.state.status = RunStatus::Blocked;
7684                self.state.event("land", format!("gave up: {e}"));
7685                self.state.save()?;
7686            }
7687        }
7688        Ok(())
7689    }
7690
7691    // -------------------------------------------------------------- helpers
7692
7693    /// Fetch or create a seat, keeping its conversation across nodes.
7694    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
7695        if let Some(existing) = self.state.seats.get(key)
7696            && existing.agent == agent
7697        {
7698            return existing.clone();
7699        }
7700        // A seat that changes agent mints its session id from the agent too,
7701        // like a handover: the old agent's uuid is already taken by the CLI.
7702        let fresh = if self.state.seats.contains_key(key) {
7703            handover_seat(key, agent, self.state.next_seat_seed())
7704        } else {
7705            SeatState::new(key, agent, self.state.seed)
7706        };
7707        self.state.seats.insert(key.to_owned(), fresh.clone());
7708        fresh
7709    }
7710
7711    /// The agent now holding seat `key`: `spec`, unless a handover moved the
7712    /// seat to another roster agent, in which case that agent. Nodes that
7713    /// continue a seat's conversation (deliberation, the votes, a reviewer's
7714    /// reconsideration) must keep talking to whoever answered it, not slip
7715    /// back to the agent that failed it.
7716    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
7717        match self.state.seats.get(key) {
7718            Some(s) if s.agent != spec.id => {
7719                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
7720            }
7721            _ => spec,
7722        }
7723    }
7724
7725    /// A candidate rendered for judging, with the leak policy applied.
7726    fn view(&self, c: &Candidate) -> CandidateView {
7727        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
7728            .unwrap_or_default();
7729        let (patch, _) = blind::sanitize_patch(
7730            &format!("candidate {} patch", c.label),
7731            &raw,
7732            &self.state.config.blind,
7733        );
7734        CandidateView {
7735            label: c.label,
7736            branch: c.branch.clone(),
7737            summary: c.summary.clone(),
7738            stat: c.stat.clone(),
7739            patch,
7740        }
7741    }
7742
7743    /// The full candidate set as prompt text, for seats with no live session.
7744    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
7745        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
7746        prompt::judge(
7747            "(see above)",
7748            &views,
7749            self.roles.judges.len(),
7750            base_short,
7751            "en",
7752        )
7753    }
7754
7755    /// The final-vote prompt with everything a seat that has no session of its
7756    /// own needs: the candidates, the seat's own ranking and reasons, and the
7757    /// anonymised deliberation it took part in (only when there was one, so a
7758    /// handed-over seat never sees more than the seat it replaces did). The
7759    /// `Final vote` heading stays first.
7760    fn vote_prompt_full(
7761        &self,
7762        j: usize,
7763        labels: &[char],
7764        language: &str,
7765        candidates: &[Candidate],
7766        base_short: &str,
7767    ) -> String {
7768        let mut text = format!(
7769            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
7770            prompt::final_vote(labels, language),
7771            self.state.instruction,
7772            self.candidate_block(candidates, base_short)
7773        );
7774        if let Some(own) = self
7775            .state
7776            .judgements
7777            .get(j)
7778            .filter(|r| !r.ranking.is_empty())
7779        {
7780            let reasons = own
7781                .reasons
7782                .iter()
7783                .map(|(k, v)| format!("- {k}: {v}"))
7784                .collect::<Vec<_>>()
7785                .join("\n");
7786            text.push_str(&format!(
7787                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
7788                own.ranking.iter().collect::<String>(),
7789                if reasons.is_empty() {
7790                    ""
7791                } else {
7792                    ", because:\n"
7793                }
7794            ));
7795        }
7796        if !self.state.deliberation.is_empty() {
7797            text.push_str("\n# What was argued before this vote\n");
7798            for t in self.transcript(&[], j) {
7799                text.push_str(&format!(
7800                    "\n## {}{}\n\n{}\n",
7801                    t.who,
7802                    if t.is_self { " (you)" } else { "" },
7803                    t.body.trim()
7804                ));
7805            }
7806        }
7807        text
7808    }
7809
7810    /// Anonymised transcript for judge `self_idx`.
7811    ///
7812    /// The initial rankings are always the opening statements. Seeding them
7813    /// only when no turn had been taken yet meant every judge after the first
7814    /// argued against a single voice instead of against the actual split — the
7815    /// disagreement is the information, so it is always on the table.
7816    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
7817        let mut turns = Vec::new();
7818        for j in &self.state.judgements {
7819            if j.ranking.is_empty() {
7820                continue;
7821            }
7822            let reasons = j
7823                .reasons
7824                .iter()
7825                .map(|(k, v)| format!("- {k}: {v}"))
7826                .collect::<Vec<_>>()
7827                .join("\n");
7828            turns.push(Turn {
7829                who: format!("Judge {} (opening ranking)", j.judge),
7830                is_self: j.judge == self_idx + 1,
7831                body: format!(
7832                    "Ranked {}{}{reasons}",
7833                    j.ranking.iter().collect::<String>(),
7834                    if reasons.is_empty() {
7835                        ""
7836                    } else {
7837                        ", because:\n"
7838                    }
7839                ),
7840            });
7841        }
7842        for t in self
7843            .state
7844            .deliberation
7845            .iter()
7846            .flat_map(|r| r.turns.iter())
7847            .chain(current)
7848        {
7849            turns.push(Turn {
7850                who: format!("Judge {}", t.judge),
7851                is_self: t.judge == self_idx + 1,
7852                body: t.body.clone(),
7853            });
7854        }
7855        turns
7856    }
7857}
7858
7859/// Does this seat still hold the context a follow-up prompt would rely on?
7860fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
7861    agent::has_session(spec.kind, seat, sessions)
7862}
7863
7864/// The next entry in `roster` after `start`, never wrapping back to the
7865/// front, whose id is not in `tried` yet.
7866///
7867/// Starts one past `start` rather than at the front of `roster`: `start` is
7868/// the seat's own original position, and a seat whose candidate slot already
7869/// sits on the roster's second entry must fall through to the third next, not
7870/// restart at the first — which is very likely a different candidate's own
7871/// agent already. Never wraps back past `start`, for the same reason: an
7872/// entry earlier in the roster than the seat's own position is almost
7873/// certainly some *other* candidate slot's own agent, and once the tail of
7874/// the roster is exhausted there are no more untried agents for *this* seat
7875/// to fall through to — the caller's fallback chain ends there, exactly as
7876/// "no further untried agents remain in the list for that seat" asks for.
7877///
7878/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
7879/// the same id twice (an operator's `roles.implementers` typo, or a
7880/// `[[agents]]` list reused across roles) must not let
7881/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
7882/// over `roster` either finds an untried id or runs out, so this always
7883/// terminates regardless of duplicates.
7884fn next_untried_in_roster<'a>(
7885    roster: &'a [AgentSpec],
7886    start: usize,
7887    tried: &BTreeSet<String>,
7888) -> Option<&'a AgentSpec> {
7889    roster
7890        .get(start + 1..)?
7891        .iter()
7892        .find(|s| !tried.contains(&s.id))
7893}
7894
7895/// The successor for a seat, shared by all four seat kinds (implement, judge,
7896/// review, advise). `others` holds the ids that *currently* occupy the other
7897/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
7898/// sees them), so roster entries beyond the seat count act as spares: a failed
7899/// seat goes to an agent no other seat holds whenever the roster permits.
7900///
7901/// `carried` is `Some` only for the review loop's carried failure history.
7902/// Order: (1) forward from `start`, never wrapping, untried, not a carried
7903/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
7904/// whole roster: untried and not another seat's occupant; (3) the plain walk
7905/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
7906/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
7907/// a better outcome than an empty seat, and it keeps the answer to "is there
7908/// a successor at all" exactly what it was before occupants were considered,
7909/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
7910/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
7911/// carried failure is a cheaper bet than doubling an agent on the panel, and
7912/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
7913/// the same round are handled one at a time, so a seat later in the batch
7914/// sees an earlier one's new occupant but not yet its own freed agent.
7915fn pick_successor<'a>(
7916    roster: &'a [AgentSpec],
7917    start: usize,
7918    tried: &BTreeSet<String>,
7919    carried: Option<&BTreeSet<String>>,
7920    others: &BTreeSet<String>,
7921) -> Option<&'a AgentSpec> {
7922    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
7923    roster
7924        .get(start + 1..)
7925        .and_then(|tail| {
7926            tail.iter()
7927                .filter(free)
7928                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
7929        })
7930        .or_else(|| {
7931            carried
7932                .is_some()
7933                .then(|| roster.iter().find(free))
7934                .flatten()
7935        })
7936        .or_else(|| match carried {
7937            Some(c) => next_for_seat(roster, start, tried, c),
7938            None => next_untried_in_roster(roster, start, tried),
7939        })
7940}
7941
7942/// The next agent for a seat that carries its failure history across rounds
7943/// (the review loop). `round_tried` is this round's own bound and starts
7944/// empty every round; `carried_failed` only decides priority.
7945///
7946/// First: an id walking forward from `start`, never wrapping, that is neither
7947/// tried this round nor failed in an earlier one. Only when that is exhausted
7948/// does it rescue: the first roster id (in roster order, so this one *does*
7949/// look before `start`) not yet tried this round, which is by then a carried
7950/// failure. Each id is rescued at most once per round, so it cannot loop.
7951fn next_for_seat<'a>(
7952    roster: &'a [AgentSpec],
7953    start: usize,
7954    round_tried: &BTreeSet<String>,
7955    carried_failed: &BTreeSet<String>,
7956) -> Option<&'a AgentSpec> {
7957    roster
7958        .get(start + 1..)?
7959        .iter()
7960        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
7961        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
7962}
7963
7964/// Where a reviewer seat starts a round: the agent that last answered it when
7965/// it is still on the roster and not marked failed, else the spec's own agent
7966/// unless it failed, else the next roster agent that has not failed, else the
7967/// spec's own agent again (the whole roster failed). Ids no longer on the
7968/// roster are ignored. An empty roster has no handover, so the spec stands.
7969fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
7970    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
7971        return spec;
7972    };
7973    let ok = |id: &str| !h.failed.contains(id);
7974    if let Some(last) = h.last_ok.as_deref()
7975        && ok(last)
7976        && let Some(s) = roster.iter().find(|s| s.id == last)
7977    {
7978        return s.clone();
7979    }
7980    if ok(&spec.id) {
7981        return spec;
7982    }
7983    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
7984    roster
7985        .get(from + 1..)
7986        .into_iter()
7987        .flatten()
7988        .chain(roster.iter())
7989        .find(|s| ok(&s.id))
7990        .cloned()
7991        .unwrap_or(spec)
7992}
7993
7994/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
7995/// seed so a CLI that mints its session id up front (`--session-id`) never
7996/// reuses the uuid the previous agent already opened under the same seat key.
7997pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
7998    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
7999}
8000
8001/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
8002/// so the classifier below and the code that builds the message cannot drift.
8003const TIMED_OUT: &str = "timed out";
8004
8005impl FailClass {
8006    /// `None` for an answer; otherwise how the turn failed.
8007    fn of(out: &AgentOutcome) -> Option<Self> {
8008        match out {
8009            AgentOutcome::Ok(_) => None,
8010            AgentOutcome::Quota(_) => Some(Self::Quota),
8011            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8012            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8013            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8014            // Not a failure of the agent. Callers check for it first; `None`
8015            // here means "no class to hand over on".
8016            AgentOutcome::ResourceBlocked(_) => None,
8017        }
8018    }
8019
8020    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
8021    fn stem_word(&self) -> &'static str {
8022        match self {
8023            Self::Quota => "quota",
8024            _ => "handover",
8025        }
8026    }
8027}
8028
8029/// The message's first line with its variable parts removed — digit runs and
8030/// path-like tokens — so "exited with Some(2)" and "exited with Some(7)" read
8031/// as one kind of failure.
8032fn failure_signature(msg: &str) -> String {
8033    let line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8034    let mut out = Vec::new();
8035    for word in line.split_whitespace() {
8036        if word.contains('/') || word.contains('\\') {
8037            out.push("<path>".to_owned());
8038            continue;
8039        }
8040        let mut w = String::new();
8041        let mut in_digits = false;
8042        for c in word.chars() {
8043            if c.is_ascii_digit() {
8044                if !in_digits {
8045                    w.push('#');
8046                }
8047                in_digits = true;
8048            } else {
8049                in_digits = false;
8050                w.push(c);
8051            }
8052        }
8053        out.push(w);
8054    }
8055    out.join(" ").chars().take(120).collect()
8056}
8057
8058/// Whether a seat that just failed with `cur` may go to the next roster agent.
8059/// A quota or a timeout always may. Any other failure may not when the agent
8060/// before it failed the same way: an error the prompt causes would otherwise
8061/// walk the whole roster. `prev` is the class of the immediately preceding
8062/// agent's failure, so a quota or timeout in between breaks the run of
8063/// identical failures by itself.
8064fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
8065    match cur {
8066        FailClass::Quota | FailClass::Timeout => true,
8067        FailClass::Other(_) => prev != Some(cur),
8068    }
8069}
8070
8071/// A short human reason for a failed outcome, for the handover record.
8072fn fail_reason(out: &AgentOutcome) -> String {
8073    match out {
8074        AgentOutcome::Ok(_) => String::new(),
8075        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
8076        AgentOutcome::Dropped(o) => format!(
8077            "the CLI dropped the stream ({})",
8078            o.dropped
8079                .as_ref()
8080                .map(|d| d.why.as_str())
8081                .unwrap_or("it ended without delivering its answer")
8082        ),
8083        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
8084            e.lines().next().unwrap_or("").chars().take(160).collect()
8085        }
8086    }
8087}
8088
8089/// Note one handover in the run: the structured record and, in the timeline,
8090/// the sentence a person reads. A quota keeps the wording it always had.
8091pub(crate) fn record_handover(
8092    state: &mut RunState,
8093    node: &str,
8094    seat: &str,
8095    from: &str,
8096    to: &str,
8097    class: &FailClass,
8098    reason: &str,
8099) {
8100    let message = if *class == FailClass::Quota {
8101        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
8102    } else {
8103        format!("{seat}: handed over {from} -> {to} ({reason})")
8104    };
8105    state.event(node, message);
8106    state.handovers.push(Handover {
8107        at: Timestamp::now(),
8108        node: node.to_owned(),
8109        seat: seat.to_owned(),
8110        from: from.to_owned(),
8111        to: to.to_owned(),
8112        reason: reason.to_owned(),
8113    });
8114}
8115
8116/// Did this reply report running a command whose own CLI never confirmed an
8117/// exit status?
8118///
8119/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
8120/// command *finished* (see that type's own doc), so this can only be `true`
8121/// for a command whose completion event carried no readable exit code — not
8122/// for one that simply is not mentioned at all. That is the one signal this
8123/// crate can read, from the same record `state.jobs` renders, about a reply
8124/// standing next to work its own CLI cannot vouch for finishing; it is
8125/// deliberately not a check on the exit code's *value* (a fixer legitimately
8126/// runs a command that fails mid-iteration before it succeeds) and not a
8127/// guess at a command still running in the background (which emits no event
8128/// at all, and so leaves no evidence here to find).
8129fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
8130    commands.iter().any(|c| c.exit_code.is_none())
8131}
8132
8133/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
8134/// trusted as a verified no-op — the adoption guard's own text-level half.
8135///
8136/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
8137/// timed out): a marker only earns the benefit of the doubt from a turn the
8138/// CLI itself vouches for finishing properly, the same house style
8139/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
8140/// *fix* report to for `commands`. A candidate that timed out, exited
8141/// non-zero, or left a command unconfirmed is read as the ordinary loss it
8142/// is, whatever prose it wrote — this returns `None` before it ever looks at
8143/// `text`. The remaining guards (the tree really is empty, the evidence is
8144/// non-empty) are the caller's: this only reads what the reply *claimed*.
8145fn verified_noop_claim(
8146    usable: bool,
8147    commands: &[agent::CommandEvidence],
8148    text: &str,
8149) -> Option<String> {
8150    (usable && !has_unconfirmed_command(commands))
8151        .then(|| verdict::verified_noop(text))
8152        .flatten()
8153}
8154
8155fn short(commit: &str) -> String {
8156    commit.chars().take(7).collect()
8157}
8158
8159fn make_executable(path: &Path) -> Result<()> {
8160    #[cfg(unix)]
8161    {
8162        use std::os::unix::fs::PermissionsExt as _;
8163        let mut perms = std::fs::metadata(path)?.permissions();
8164        perms.set_mode(0o755);
8165        std::fs::set_permissions(path, perms)?;
8166    }
8167    #[cfg(not(unix))]
8168    {
8169        let _ = path;
8170    }
8171    Ok(())
8172}
8173
8174/// What every seat in one batch shares: where the answers are attributed, the
8175/// prompt overlay they inherit, and the build cache they are told to use.
8176///
8177/// A struct rather than four more parameters: `wave` also needs the run's
8178/// state (to record who is answering right now) and the attempt number, and
8179/// eight positional arguments is both unreadable and a clippy error.
8180struct WaveCtx<'a> {
8181    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
8182    /// paid for it.
8183    run: &'a str,
8184    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
8185    node: &'a str,
8186    prompts: &'a Prompts,
8187    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
8188    cache: Option<&'a Path>,
8189    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
8190    /// `JobRecord::round`. `None` for every other node.
8191    round: Option<usize>,
8192    /// Carry each seat's failed-agent history across waves (the review loop
8193    /// only): start-of-round priority and handover choice read
8194    /// [`RunState::seat_history`], and every answer or failure writes it.
8195    carry_seats: bool,
8196    /// The run's park / interrupt requests, so a wait for a shared resource
8197    /// can be abandoned the moment the operator asks to park.
8198    pauses: Option<(&'a Pause, &'a Pause)>,
8199}
8200
8201/// Run one job, honouring the parallelism budget.
8202async fn run_one(
8203    job: SeatJob,
8204    sem: Arc<Semaphore>,
8205    ctx: &WaveCtx<'_>,
8206    state: &mut RunState,
8207    attempt: usize,
8208) -> (SeatState, AgentOutcome) {
8209    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
8210        .await
8211        .pop()
8212        .expect("one job in, one result out");
8213    (seat, out)
8214}
8215
8216/// Run every job concurrently, capped by the semaphore, preserving order.
8217///
8218/// Every seat in the batch is recorded into [`RunState::active`] before the
8219/// wave starts and cleared as each answer lands, so the run's own record says
8220/// who is still being waited on rather than only who finished.
8221async fn wave(
8222    jobs: Vec<SeatJob>,
8223    sem: Arc<Semaphore>,
8224    ctx: &WaveCtx<'_>,
8225    state: &mut RunState,
8226    attempt: usize,
8227) -> Vec<(usize, SeatState, AgentOutcome)> {
8228    let WaveCtx {
8229        run,
8230        node,
8231        prompts,
8232        cache,
8233        round,
8234        carry_seats: _,
8235        pauses,
8236    } = *ctx;
8237    for job in &jobs {
8238        state.seat_started(node, &job.seat.key, job.timeout, attempt);
8239    }
8240    if let Err(e) = state.save() {
8241        // A failed persist of "who is answering right now" must not abort the
8242        // wave: the seats are already being asked, and the alternative is
8243        // losing the answers to save a status line nobody may even be
8244        // watching.
8245        tracing::warn!("could not persist in-progress seats: {e:#}");
8246    }
8247    // Hold the shared build cache's lease for the whole batch, not per job:
8248    // several candidates (an implement wave) or a fixer legitimately share
8249    // one cache concurrently within this run, and that stays untouched — a
8250    // single lease taken once for the whole wave and released once it is
8251    // done is what stops a *different* borrower (another run's own wave, its
8252    // e2e/gate, a human's `magi review`) from interleaving a build into the
8253    // same directory while this one is in flight. Best-effort, not
8254    // all-or-nothing: a wave that cannot get the lease within its own
8255    // longest job's budget still runs — an hour of paid implementer calls is
8256    // not thrown away over cache contention — but every write-allowed seat
8257    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
8258    // below), the same fallback a read-only seat always gets, rather than
8259    // building into a directory this run was never granted. The identity
8260    // record is still invalidated below either way, so the next tracked
8261    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
8262    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
8263    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
8264    // never borrowed from a job's agent budget (and never charged to it either,
8265    // see below). The agent's clock starts in `agent::invoke`, at spawn.
8266    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
8267    let cache_guard = if let Some(cache_dir) = cache {
8268        if jobs_had_a_writer {
8269            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
8270            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
8271            match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
8272            {
8273                Ok(Some(g)) => Some(g),
8274                // Parking: leave without spawning anything. The run parks at
8275                // the next boundary and the seats are asked afresh on resume.
8276                Ok(None) => {
8277                    state.resource_block = Some(crate::run::ResourceBlock {
8278                        node: node.to_owned(),
8279                        kind: "shared build cache".to_owned(),
8280                        detail: "the run is parking".to_owned(),
8281                        at: Timestamp::now(),
8282                    });
8283                    return block_wave(
8284                        jobs,
8285                        state,
8286                        node,
8287                        attempt,
8288                        "parked while waiting for the shared build cache",
8289                    );
8290                }
8291                // An unreadable lease is not contention: keep the old
8292                // fallback (no shared cache for this wave's seats).
8293                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
8294                Err(e) => {
8295                    // Writing seats may not build into a cache this run does
8296                    // not hold, and an agent that cannot be given one is not
8297                    // asked at all: the run stops resumably instead.
8298                    let why = format!("{e:#}");
8299                    state.resource_block = Some(crate::run::ResourceBlock {
8300                        node: node.to_owned(),
8301                        kind: "shared build cache".to_owned(),
8302                        detail: why.clone(),
8303                        at: Timestamp::now(),
8304                    });
8305                    state.event(
8306                        "resource",
8307                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
8308                    );
8309                    return block_wave(jobs, state, node, attempt, &why);
8310                }
8311            }
8312        } else {
8313            None
8314        }
8315    } else {
8316        None
8317    };
8318    // Every job keeps its whole budget: the wait above was not the agent's.
8319    // Restart the seats' clocks so "started" means "being spawned now".
8320    state.seats_spawning(node, attempt);
8321    if let Err(e) = state.save() {
8322        tracing::warn!("could not persist the start of a wave: {e:#}");
8323    }
8324    // Queueing behind this wave's own seats for a parallelism slot is normal
8325    // (each of them is bounded by its own agent timeout); only a wait beyond
8326    // all of them plus the resource limit means the slots never came back.
8327    let slot_limit = resource_limit
8328        + jobs
8329            .iter()
8330            .map(|j| j.timeout)
8331            .max()
8332            .unwrap_or_default()
8333            .saturating_mul(jobs.len() as u32);
8334    let mut set = tokio::task::JoinSet::new();
8335    // Seats announce themselves here the moment they hold a parallelism
8336    // permit, i.e. just before their CLI is spawned.
8337    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
8338    let overlay = prompts.overlay(node);
8339    for (i, mut job) in jobs.into_iter().enumerate() {
8340        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
8341        if cache.is_some() {
8342            job.prompt.push('\n');
8343            job.prompt
8344                .push_str(&prompt::build_cache_note(node, job.allow_write));
8345        }
8346        let sem = Arc::clone(&sem);
8347        let run = run.to_owned();
8348        let node = node.to_owned();
8349        // Only implementers were told about the task's attachments, so only
8350        // their seats get the directory widened for reading.
8351        let attachments = if node == "implement" {
8352            state.attachments.clone()
8353        } else {
8354            Vec::new()
8355        };
8356        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
8357        // `prompt::build_cache_note`'s doc for why setting it anyway is
8358        // exactly how a sandboxed reviewer's write refusal got reported as a
8359        // defect in the patch, not a property of its own seat. And a
8360        // write-allowed one is handed it only when the lease above was
8361        // actually acquired: a wave that could not get it (`cache_guard` is
8362        // `None`, see its own comment) must not send seats to build into a
8363        // directory this run does not hold - that is the exact concurrent,
8364        // unmanaged-write race this module exists to prevent, not something
8365        // "proceeding anyway" is allowed to reintroduce.
8366        let cache = cache
8367            .filter(|_| job.allow_write && cache_guard.is_some())
8368            .map(Path::to_path_buf);
8369        let spawned_tx = spawned_tx.clone();
8370        set.spawn(async move {
8371            // A wait for a parallelism slot is scheduling, not the agent: it
8372            // is bounded by its own limit and ends as a resource block, never
8373            // as a timeout of the agent that was not started.
8374            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
8375                Ok(p) => p,
8376                Err(_) => {
8377                    return (
8378                        i,
8379                        job.seat,
8380                        AgentOutcome::ResourceBlocked(format!(
8381                            "no parallel slot became free within {}s",
8382                            slot_limit.as_secs()
8383                        )),
8384                    );
8385                }
8386            };
8387            spawned_tx.send(job.seat.key.clone()).ok();
8388            let mut seat = job.seat;
8389            let out = agent::invoke(
8390                &job.spec,
8391                &mut seat,
8392                &Invocation {
8393                    cwd: &job.cwd,
8394                    prompt: &job.prompt,
8395                    timeout: job.timeout,
8396                    allow_write: job.allow_write,
8397                    unsandboxed: false,
8398                    sessions: job.sessions,
8399                    artifacts: &job.artifacts,
8400                    stem: &job.stem,
8401                    run: &run,
8402                    node: &node,
8403                    cache_dir: cache.as_deref(),
8404                    attachments: &attachments,
8405                    writable: &[],
8406                },
8407            )
8408            .await;
8409            let out = match out {
8410                Ok(o) if o.usable() => AgentOutcome::Ok(o),
8411                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
8412                // Billed work the CLI failed to hand over is not an ordinary
8413                // failure, but its text is the CLI's raw error JSON, not an
8414                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
8415                // read it as one by forgetting to check. `usable()` is always
8416                // false here (dropped implies an empty response), so this has
8417                // to be checked before the catch-all `Failed` below or the
8418                // one shape this exists for is lost with the rest.
8419                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
8420                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
8421                Ok(o) => AgentOutcome::Failed(format!(
8422                    "exited with {:?} and no usable output",
8423                    o.exit_code
8424                )),
8425                Err(e) => AgentOutcome::Failed(e.to_string()),
8426            };
8427            (i, seat, out)
8428        });
8429    }
8430    drop(spawned_tx);
8431    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
8432    loop {
8433        let joined = tokio::select! {
8434            Some(key) = spawned_rx.recv() => {
8435                // The clock of this seat starts now, not when it queued.
8436                state.seat_spawned(&key);
8437                if let Err(e) = state.save() {
8438                    tracing::warn!("could not persist a seat's start: {e:#}");
8439                }
8440                continue;
8441            }
8442            joined = set.join_next() => joined,
8443        };
8444        let Some(joined) = joined else {
8445            break;
8446        };
8447        let (i, seat, out) = match joined {
8448            Ok(v) => v,
8449            // No seat to clear: a panicked task never reported which one it
8450            // was. The defensive sweep below this loop is what stops that
8451            // seat's `active` entry from surviving forever.
8452            Err(e) => {
8453                tracing::error!("agent task panicked: {e}");
8454                continue;
8455            }
8456        };
8457        state.seat_finished(&seat.key);
8458        if let AgentOutcome::ResourceBlocked(why) = &out
8459            && state.resource_block.is_none()
8460        {
8461            state.resource_block = Some(crate::run::ResourceBlock {
8462                node: node.to_owned(),
8463                kind: "parallel slot".to_owned(),
8464                detail: why.clone(),
8465                at: Timestamp::now(),
8466            });
8467            state.event("resource", format!("{node}: no agent was started - {why}"));
8468        }
8469        record_jobs(state, node, round, &seat.key, &out);
8470        if let Err(e) = state.save() {
8471            tracing::warn!("could not persist a seat's completion: {e:#}");
8472        }
8473        if collected.len() <= i {
8474            collected.resize_with(i + 1, || None);
8475        }
8476        collected[i] = Some((i, seat, out));
8477    }
8478    // Belt-and-braces for the panic branch above: every seat this exact batch
8479    // started shares this `(node, attempt)` pair, and every seat that finished
8480    // normally already cleared itself, so anything left tagged with it here
8481    // can only be a panicked task's leftover. Cleared unconditionally rather
8482    // than left to read as still answering forever.
8483    if state
8484        .active
8485        .values()
8486        .any(|a| a.node == node && a.attempt == attempt)
8487    {
8488        state
8489            .active
8490            .retain(|_, a| !(a.node == node && a.attempt == attempt));
8491        if let Err(e) = state.save() {
8492            tracing::warn!("could not persist the end of a wave: {e:#}");
8493        }
8494    }
8495    // Whether or not the lease above was actually held, several worktrees
8496    // may just have built into the cache with nothing here able to name one
8497    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
8498    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
8499    // clean it might not have strictly needed; trusting a stale match would
8500    // cost it a wrong answer.
8501    if let Some(cache_dir) = cache
8502        && jobs_had_a_writer
8503    {
8504        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
8505    }
8506    if let Some(guard) = cache_guard {
8507        guard.release();
8508    }
8509    collected.into_iter().flatten().collect()
8510}
8511
8512/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
8513/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
8514/// this is data collection, not the fix-specific completion contract in
8515/// [`Runner::continue_fix_report`], and applies regardless of which node
8516/// asked.
8517///
8518/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
8519/// evidence from; `Failed` does not, and correctly contributes nothing — a
8520/// timeout or crash is not itself evidence about a command the seat may have
8521/// started.
8522fn record_jobs(
8523    state: &mut RunState,
8524    node: &str,
8525    round: Option<usize>,
8526    seat: &str,
8527    out: &AgentOutcome,
8528) {
8529    let commands: &[agent::CommandEvidence] = match out {
8530        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
8531        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
8532    };
8533    let checked_at = Timestamp::now();
8534    for c in commands {
8535        state.jobs.push(JobRecord {
8536            node: node.to_owned(),
8537            round,
8538            seat: seat.to_owned(),
8539            id: c.id.clone(),
8540            description: c.description.clone(),
8541            checked_at,
8542            status: match c.exit_code {
8543                Some(0) => JobStatus::Completed,
8544                Some(_) => JobStatus::Failed,
8545                None => JobStatus::Unknown,
8546            },
8547            exit_code: c.exit_code,
8548            result_summary: c.result_summary.clone(),
8549            source: c.source.clone(),
8550        });
8551    }
8552}
8553
8554/// Is a review round clean, given how many reviewer seats answered against
8555/// how many the round expected?
8556///
8557/// A seat that never answered (timeout, crash, unparsable output) is not a
8558/// seat that read the patch and found nothing — treating it as such is
8559/// exactly the bug this function exists to close. Under the default `block`
8560/// policy a missing seat can never be clean; `warn` still requires the seats
8561/// that *did* answer to have found nothing blocking and verification to be
8562/// green.
8563///
8564/// `quota_missing` narrows that `block` default for exactly one cause of
8565/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
8566/// a session limit lifts by the very next round buys nothing — the seat is
8567/// asked again with the same quota — so once every missing seat is accounted
8568/// for by a quota loss (and at least one seat *did* answer, so a decision has
8569/// something to rest on) the round is decided on the panel that could answer,
8570/// same as `warn` would. A panel that lost every seat to quota is not
8571/// decided here: `answered == 0` falls through to the existing `block`
8572/// fallback so a fully collapsed panel still waits rather than landing on no
8573/// review at all.
8574fn round_is_clean(
8575    blocking: usize,
8576    e2e_ok: bool,
8577    answered: usize,
8578    expected: usize,
8579    quota_missing: usize,
8580    policy: IncompleteReviewPolicy,
8581) -> bool {
8582    if blocking != 0 || !e2e_ok {
8583        return false;
8584    }
8585    if answered == expected || policy == IncompleteReviewPolicy::Warn {
8586        return true;
8587    }
8588    answered > 0 && expected - answered <= quota_missing
8589}
8590
8591/// The review loop's own conclusion, derived entirely from its persisted
8592/// round records and the round budget that produced them — never from
8593/// `status`, so a reentry (or `gate`/`merge` reading it independently)
8594/// recomputes the identical answer regardless of what an earlier node in the
8595/// same walk, or a previous walk, did to `status`.
8596///
8597/// `None` while more rounds remain to try, including when review never ran
8598/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
8599/// gone clean, or the budget is spent, or the tree has stopped moving (see
8600/// [`STAGNANT_LIMIT`]), the answer is one of two things:
8601///
8602/// - An incomplete panel that raised nothing is missing input, not a
8603///   verified tree — never a hand-off candidate, whatever verification said
8604///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
8605/// - Otherwise, green e2e on the last round hands off (see
8606///   [`Runner::stop_reviewing`]); red e2e blocks.
8607///
8608/// A last round whose own verification is still `ResourceBlocked` — magi
8609/// itself never got a command to run, not evidence the patch is broken —
8610/// is neither: this returns `None` for it too, the same as "more rounds
8611/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
8612/// handling of that shape) instead of this cheap recomputation guessing a
8613/// verdict a real attempt never produced.
8614fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
8615    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
8616        return Some(RunStatus::Gating);
8617    }
8618    let last = reviews.last()?;
8619    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
8620    if reviews.len() < max_rounds && !stagnant {
8621        return None;
8622    }
8623    if last.incomplete() && last.blocking == 0 {
8624        return Some(RunStatus::Blocked);
8625    }
8626    if last.e2e_status() == E2eStatus::ResourceBlocked {
8627        return None;
8628    }
8629    Some(if last.e2e.iter().all(CommandOutcome::ok) {
8630        RunStatus::Gating
8631    } else {
8632        RunStatus::Blocked
8633    })
8634}
8635
8636/// How long a re-ask may take, given the budget the first attempt had.
8637///
8638/// A `nudged` retry is a request to restate an answer the seat has already
8639/// worked out: it carries no new work, so it does not deserve the original
8640/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
8641/// 133 seconds while a third sat for over ten minutes on a resumed session
8642/// holding 410 KB of prior output - and because the retry had inherited the
8643/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
8644/// judging round whose other seats were long finished.
8645///
8646/// A quarter of the budget, with a floor so that a deliberately short timeout
8647/// does not collapse to nothing. A retry that re-sends the whole prompt
8648/// (because the seat kept no context) is the original job again, and keeps the
8649/// original budget.
8650pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
8651    if nudged {
8652        (full / 4).max(Duration::from_secs(120)).min(full)
8653    } else {
8654        full
8655    }
8656}
8657
8658/// Run a wave and parse each reply, re-asking the seats whose reply was
8659/// unusable.
8660///
8661/// The re-ask is a nudge rather than the whole prompt again when the seat still
8662/// holds its conversation, which is the difference between a cheap retry and
8663/// paying for the entire candidate set twice.
8664///
8665/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
8666/// successor in `roster` is handed to it instead of being re-asked: the
8667/// handover is the retry. A seat with no successor left (a single-agent
8668/// roster, the roster's tail) is nudged as before, up to `retries` times. So
8669/// the asks to one seat in one node number at most
8670/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
8671/// is asked once (a dropped stream is nudged first), and only the last agent
8672/// of the chain gets the `retries` same-agent nudges. Each roster agent is
8673/// tried at most once per seat, walking forward from the seat's own position and never wrapping
8674/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
8675/// other failure stops the chain when the previous agent failed the same way
8676/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
8677/// [`has_context`] is false and the job's own full prompt and full budget are
8678/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
8679/// intermediate ones are not losses) and is returned as a failure like any
8680/// other absent seat — the caller decides whether the panel still has a
8681/// quorum. An empty `roster` disables handover: failures are nudged as they
8682/// always were, and a quota is simply lost. A reply that fails to parse or
8683/// validate is the prompt's doing and is only ever nudged, never handed over.
8684///
8685/// The returned [`SeatState`] names the agent that answered (or tried last).
8686#[allow(clippy::too_many_arguments)]
8687async fn ask_json_wave<T>(
8688    jobs: Vec<SeatJob>,
8689    sem: Arc<Semaphore>,
8690    retries: usize,
8691    roster: &[AgentSpec],
8692    ctx: &WaveCtx<'_>,
8693    losses: &mut Vec<QuotaLoss>,
8694    state: &mut RunState,
8695    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
8696) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
8697where
8698    T: serde::de::DeserializeOwned + Send + 'static,
8699{
8700    ask_wave_with(
8701        jobs,
8702        sem,
8703        retries,
8704        roster,
8705        ctx,
8706        losses,
8707        state,
8708        &|text: &str| {
8709            let v = verdict::extract_json::<T>(text)?;
8710            validate(&v)?;
8711            Ok(v)
8712        },
8713    )
8714    .await
8715}
8716
8717/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
8718/// a node whose answer is prose (deliberation) shares the same handover,
8719/// failure classification, quota bookkeeping and bounds instead of a copy.
8720///
8721/// A seat handed to another roster agent is sent the job's `handover` prompt
8722/// (when it has one) rather than `prompt`: the new agent has no session, so a
8723/// resume-style prompt would be incomplete. That holds for the handover ask
8724/// and for every nudge to that agent whose `has_context` is false.
8725#[allow(clippy::too_many_arguments)]
8726async fn ask_wave_with<T>(
8727    jobs: Vec<SeatJob>,
8728    sem: Arc<Semaphore>,
8729    retries: usize,
8730    roster: &[AgentSpec],
8731    ctx: &WaveCtx<'_>,
8732    losses: &mut Vec<QuotaLoss>,
8733    state: &mut RunState,
8734    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
8735) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
8736where
8737    T: Send + 'static,
8738{
8739    let n = jobs.len();
8740    let originals: Vec<SeatJob> = jobs;
8741    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
8742    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
8743    // Nudges each seat's *current* agent has taken — 0 for a first-ask
8744    // answer, N once it has gone through N nudges. Read back once this
8745    // returns, so a caller building a history record (`ReviewRecord`) can
8746    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
8747    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
8748    // field's own doc.
8749    let mut nudges: Vec<usize> = vec![0; n];
8750    // The agent now occupying each seat, the ids it has already been through,
8751    // where in the roster the walk began, the class of the last failure, and
8752    // the stem word of a handover not yet asked (full prompt, full budget).
8753    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
8754    let mut tried: Vec<BTreeSet<String>> = specs
8755        .iter()
8756        .map(|s| BTreeSet::from([s.id.clone()]))
8757        .collect();
8758    let starts: Vec<usize> = specs
8759        .iter()
8760        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
8761        .collect();
8762    let carry = ctx.carry_seats && !roster.is_empty();
8763    // Carried across rounds: ids that failed the seat earlier, and how the
8764    // last failure went (so a repeat of it is not handed over again).
8765    let carried: Vec<BTreeSet<String>> = originals
8766        .iter()
8767        .map(|j| {
8768            state
8769                .seat_history
8770                .get(&j.seat.key)
8771                .filter(|_| carry)
8772                .map(|h| h.failed.clone())
8773                .unwrap_or_default()
8774        })
8775        .collect();
8776    let mut prev: Vec<Option<FailClass>> = originals
8777        .iter()
8778        .map(|j| {
8779            state
8780                .seat_history
8781                .get(&j.seat.key)
8782                .filter(|_| carry)
8783                .and_then(|h| h.last_fail.clone())
8784        })
8785        .collect();
8786    let next_agent =
8787        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
8788            let others: BTreeSet<String> = specs
8789                .iter()
8790                .enumerate()
8791                .filter(|(j, _)| *j != i)
8792                .map(|(_, s)| s.id.clone())
8793                .collect();
8794            pick_successor(
8795                roster,
8796                starts[i],
8797                tried,
8798                carry.then(|| &carried[i]),
8799                &others,
8800            )
8801            .cloned()
8802        };
8803    let mut fresh: Vec<Option<String>> = vec![None; n];
8804    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
8805    let mut pending: Vec<usize> = (0..n).collect();
8806
8807    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
8808    // asks: an agent with a successor is asked once and handed over, and only
8809    // a seat with no successor spends `retries` nudges on the same agent. This
8810    // only guarantees the loop's own termination whatever those say.
8811    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
8812    for round in 0..max_rounds {
8813        if pending.is_empty() {
8814            break;
8815        }
8816        let mut batch = Vec::with_capacity(pending.len());
8817        let mut renudged: Vec<&str> = Vec::new();
8818        for &i in &pending {
8819            let src = &originals[i];
8820            // A seat now held by another agent than the job named has no
8821            // session of its own: it gets the full-context prompt whenever
8822            // it is asked in full (the handover ask, a nudge it cannot
8823            // resume).
8824            let full: &str = match &src.handover {
8825                Some(h) if specs[i].id != src.spec.id => h,
8826                _ => &src.prompt,
8827            };
8828            // The prompt and the budget are one decision: a nudge restates
8829            // finished work, a re-sent prompt redoes it.
8830            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
8831                (
8832                    full.to_owned(),
8833                    src.timeout,
8834                    format!("{}-{word}-{}", src.stem, specs[i].id),
8835                )
8836            } else if nudges[i] == 0 {
8837                (full.to_owned(), src.timeout, src.stem.clone())
8838            } else {
8839                renudged.push(src.seat.key.as_str());
8840                let why = done[i]
8841                    .as_ref()
8842                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
8843                    .unwrap_or_else(|| "no parsable answer".to_owned());
8844                let nudge = prompt::nudge(&why);
8845                let nudged = has_context(&specs[i], &seats[i], src.sessions);
8846                let prompt = if nudged {
8847                    nudge
8848                } else {
8849                    format!("{full}\n\n---\n\n{nudge}")
8850                };
8851                (
8852                    prompt,
8853                    retry_budget(src.timeout, nudged),
8854                    format!("{}-retry{}", src.stem, nudges[i]),
8855                )
8856            };
8857            batch.push(SeatJob {
8858                spec: specs[i].clone(),
8859                seat: seats[i].clone(),
8860                cwd: src.cwd.clone(),
8861                prompt,
8862                timeout,
8863                allow_write: src.allow_write,
8864                sessions: src.sessions,
8865                artifacts: src.artifacts.clone(),
8866                stem,
8867                handover: None,
8868            });
8869        }
8870
8871        if !renudged.is_empty() {
8872            state.event(
8873                ctx.node,
8874                format!("retry {round}: re-asking {}", renudged.join(", ")),
8875            );
8876        }
8877        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
8878        let mut still = Vec::new();
8879        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
8880            seats[i] = seat;
8881            // No agent was asked: nothing to hand over, nudge, remember or
8882            // charge. The caller sees `resource_block` and stops the node.
8883            if let AgentOutcome::ResourceBlocked(why) = &out {
8884                done[i] = Some(Err(anyhow::anyhow!("{why}")));
8885                continue;
8886            }
8887            let class = FailClass::of(&out);
8888            // A dropped stream is nudged first (the conversation is still
8889            // there to pick up); only a seat whose nudges are spent hands over.
8890            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
8891                && nudges[i] < retries
8892                && !roster.is_empty();
8893            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
8894                let next = should_hand_over(prev[i].as_ref(), &cur)
8895                    .then(|| next_agent(i, &tried[i], &specs))
8896                    .flatten();
8897                if carry {
8898                    let h = state
8899                        .seat_history
8900                        .entry(originals[i].seat.key.clone())
8901                        .or_default();
8902                    h.failed.insert(specs[i].id.clone());
8903                    h.last_fail = Some(cur.clone());
8904                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
8905                        h.last_ok = None;
8906                    }
8907                    // Saved before the next agent is asked, so a restart in
8908                    // between does not forget who failed.
8909                    if let Err(e) = state.save() {
8910                        tracing::warn!("could not persist a seat's failure history: {e:#}");
8911                    }
8912                }
8913                if let Some(next) = next {
8914                    record_handover(
8915                        state,
8916                        ctx.node,
8917                        &originals[i].seat.key,
8918                        &specs[i].id,
8919                        &next.id,
8920                        &cur,
8921                        &fail_reason(&out),
8922                    );
8923                    tried[i].insert(next.id.clone());
8924                    prev[i] = Some(cur.clone());
8925                    seats[i] =
8926                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
8927                    specs[i] = next;
8928                    fresh[i] = Some(cur.stem_word().to_owned());
8929                    nudges[i] = 0;
8930                    done[i] = Some(Err(anyhow::anyhow!(
8931                        "handed over after: {}",
8932                        fail_reason(&out)
8933                    )));
8934                    still.push(i);
8935                    continue;
8936                }
8937            }
8938            if carry
8939                && !nudge_first
8940                && let Some(cur) = class.clone()
8941            {
8942                // The chain ended here (no successor, or a repeated failure).
8943                let h = state
8944                    .seat_history
8945                    .entry(originals[i].seat.key.clone())
8946                    .or_default();
8947                h.failed.insert(specs[i].id.clone());
8948                h.last_fail = Some(cur);
8949            }
8950            let parsed = match out {
8951                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
8952                AgentOutcome::Quota(o) => {
8953                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
8954                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
8955                }
8956                // Not a parseable answer: the nudge loop re-asks it, which is
8957                // exactly what a dropped stream needs. Just don't hand its raw
8958                // error JSON to `extract_json`.
8959                AgentOutcome::Dropped(o) => {
8960                    let why = o
8961                        .dropped
8962                        .as_ref()
8963                        .map(|d| d.why.as_str())
8964                        .unwrap_or("the CLI ended the stream without delivering its answer");
8965                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
8966                }
8967                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
8968                    Err(anyhow::anyhow!(e))
8969                }
8970            };
8971            let quota = class == Some(FailClass::Quota);
8972            let failed = parsed.is_err();
8973            done[i] = Some(parsed);
8974            if carry && !failed {
8975                let h = state
8976                    .seat_history
8977                    .entry(originals[i].seat.key.clone())
8978                    .or_default();
8979                h.failed.remove(&specs[i].id);
8980                h.last_ok = Some(specs[i].id.clone());
8981                h.last_fail = None;
8982            }
8983            // Do not re-ask a rate-limited seat (quota) — a retry is known to
8984            // fail the same way; and never re-ask a seat that already parsed.
8985            // A failed agent that still has a successor is not re-asked
8986            // either: the handover was its remedy and has just been refused
8987            // (the chain stops on a repeated failure class). A seat with no
8988            // successor left (a single-agent roster, the roster's tail, or an
8989            // empty roster) keeps the same-agent nudge, bounded by `retries`.
8990            let agent_failure = class.is_some()
8991                && !nudge_first
8992                && !roster.is_empty()
8993                && next_agent(i, &tried[i], &specs).is_some();
8994            if failed && !quota && !agent_failure && nudges[i] < retries {
8995                nudges[i] += 1;
8996                still.push(i);
8997            }
8998        }
8999        pending = still;
9000    }
9001
9002    // One loss per seat whose chain ended on a quota: the intermediate ones
9003    // were absorbed by a handover and are not losses.
9004    for (i, q) in last_quota.into_iter().enumerate() {
9005        if let Some(reset) = q {
9006            losses.push(QuotaLoss {
9007                seat: originals[i].seat.key.clone(),
9008                node: ctx.node.to_owned(),
9009                at: Timestamp::now(),
9010                reset,
9011            });
9012        }
9013    }
9014
9015    seats
9016        .into_iter()
9017        .zip(done)
9018        .zip(nudges)
9019        .map(|((seat, res), attempts)| {
9020            (
9021                seat,
9022                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
9023                attempts,
9024            )
9025        })
9026        .collect()
9027}
9028
9029/// Did the wave end without asking any agent (see [`block_wave`])?
9030fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
9031    results
9032        .iter()
9033        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
9034}
9035
9036/// Settle a wave that never spawned anything: every job answers
9037/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
9038/// else about the seat (session, history, quota) is touched.
9039fn block_wave(
9040    jobs: Vec<SeatJob>,
9041    state: &mut RunState,
9042    node: &str,
9043    attempt: usize,
9044    why: &str,
9045) -> Vec<(usize, SeatState, AgentOutcome)> {
9046    state.resource_wait = None;
9047    state
9048        .active
9049        .retain(|_, a| !(a.node == node && a.attempt == attempt));
9050    if let Err(e) = state.save() {
9051        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
9052    }
9053    jobs.into_iter()
9054        .enumerate()
9055        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
9056        .collect()
9057}
9058
9059/// Acquire the shared build cache's lease, waiting out contention within
9060/// `budget` (never past it — see AGENTS.md's build-cache section on why an
9061/// unbounded wait is never acceptable).
9062///
9063/// A first, non-blocking check happens before ever waiting; if it finds the
9064/// lease busy, that fact is logged as a `verify` event *and* flushed with
9065/// [`RunState::save`] immediately — not only once the wait finally succeeds
9066/// or gives up — so a `magi show` run by a different process while this one
9067/// is still waiting reads a `run.json` that says so, rather than whatever it
9068/// looked like before the wait started. The same applies to the terminal
9069/// failure: logged and saved before this returns `Err`, so a caller that
9070/// could not get the lease at all still leaves a legible record of why.
9071async fn acquire_cache_lease(
9072    state: &mut RunState,
9073    cache_dir: &Path,
9074    owner: &crate::cache::Owner,
9075    budget: Duration,
9076    context: &str,
9077    cancel: &(dyn Fn() -> bool + Sync),
9078) -> Result<Option<crate::cache::Guard>> {
9079    let home = crate::run::home();
9080    let started = Instant::now();
9081    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
9082        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
9083        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
9084        Err(e) => {
9085            state.event(
9086                "verify",
9087                format!("{context}: could not check the shared build cache: {e:#}"),
9088            );
9089            if let Err(e2) = state.save() {
9090                tracing::warn!("could not persist a cache-check failure: {e2:#}");
9091            }
9092            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
9093        }
9094    };
9095    state.event(
9096        "verify",
9097        format!(
9098            "{context}: waiting for the shared build cache at {} ({})",
9099            cache_dir.display(),
9100            busy.describe()
9101        ),
9102    );
9103    // Seats of this node are waiting, not running: say so, so the countdown a
9104    // reader sees only ever measures an agent that has been spawned.
9105    state.resource_wait = Some(crate::run::ResourceWait {
9106        node: context.to_owned(),
9107        kind: "shared build cache".to_owned(),
9108        holder: busy.describe(),
9109        since: Timestamp::now(),
9110        limit_secs: budget.as_secs(),
9111    });
9112    if let Err(e) = state.save() {
9113        tracing::warn!("could not persist a cache wait: {e:#}");
9114    }
9115    let remaining = budget.saturating_sub(started.elapsed());
9116    let waited = crate::cache::wait_for_unless(
9117        &home,
9118        cache_dir,
9119        owner,
9120        remaining,
9121        Duration::from_secs(5),
9122        cancel,
9123    )
9124    .await;
9125    state.resource_wait = None;
9126    match waited {
9127        Ok(g) => Ok(g),
9128        Err(e) => {
9129            state.event("verify", format!("{context}: {e:#}"));
9130            if let Err(e2) = state.save() {
9131                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
9132            }
9133            Err(e)
9134        }
9135    }
9136}
9137
9138/// The lease state could not be read at all (unreadable lease file, a
9139/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
9140/// known to be contended, so a wave degrades to running without a shared cache
9141/// instead of stopping on a hold that [`crate::triage`] could never release.
9142#[derive(Debug)]
9143struct CacheCheckError(String);
9144
9145impl std::fmt::Display for CacheCheckError {
9146    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
9147        f.write_str(&self.0)
9148    }
9149}
9150
9151impl std::error::Error for CacheCheckError {}
9152
9153/// Run `body` — a verify command batch — while holding the shared build
9154/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
9155/// never interleave with another borrower's build against the same
9156/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
9157/// timeout, or a human's own `magi review`. See the `cache` module doc for
9158/// why this matters more than Cargo's own per-target locking covers — two
9159/// *different* worktrees building the same package name/version into one
9160/// cache directory is a staleness bug, not a lock contention one.
9161///
9162/// The wait for the lease is carved out of `budget`, never on top of it —
9163/// `body` is handed whatever is left, so a caller's own node timeout is the
9164/// only clock involved, exactly what AGENTS.md's build-cache section asks
9165/// for ("never an unbounded wait"). When `cache_dir` is `None` — no shared
9166/// cache configured at all — this is a pass-through: `body` runs with the
9167/// full budget and nothing is leased.
9168///
9169/// A lease that cannot be acquired within `budget` is reported as a single
9170/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
9171/// verification — the same shape a spawn failure already takes in
9172/// [`run_commands`], so a caller need not special-case it.
9173#[allow(clippy::too_many_arguments)]
9174async fn with_cache_lease<'s, F, Fut>(
9175    state: &'s mut RunState,
9176    cache_dir: Option<&Path>,
9177    node: &str,
9178    seat: &str,
9179    worktree: &Path,
9180    head: &str,
9181    budget: Duration,
9182    context: &str,
9183    body: F,
9184) -> (Vec<CommandOutcome>, bool)
9185where
9186    F: FnOnce(&'s mut RunState, Duration) -> Fut,
9187    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
9188{
9189    let Some(cache_dir) = cache_dir else {
9190        let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
9191        return (outcomes, retried);
9192    };
9193    let home = crate::run::home();
9194    let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
9195    let started = Instant::now();
9196    let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context, &|| false)
9197        .await
9198        .and_then(|g| {
9199            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
9200        }) {
9201        Ok(g) => g,
9202        Err(e) => {
9203            return (
9204                vec![CommandOutcome {
9205                    command: "(waiting for the shared build cache)".to_owned(),
9206                    code: None,
9207                    output_tail: e.to_string(),
9208                    duration_ms: started.elapsed().as_millis() as u64,
9209                    resource_blocked: true,
9210                }],
9211                false,
9212            );
9213        }
9214    };
9215    let identity = crate::cache::Identity::new(worktree, head);
9216    if let Err(e) = crate::cache::ensure_fresh(
9217        &home,
9218        cache_dir,
9219        &identity,
9220        &state.config.verify.cargo_manifests(),
9221    ) {
9222        // A failed freshness check means this process cannot vouch for what
9223        // is sitting in the cache right now - on Windows this is exactly the
9224        // "a stale test executable is still locked, `cargo clean -p` cannot
9225        // remove it" case the evidence log records. Running verify anyway
9226        // and reporting whatever it says would let a result nobody can trust
9227        // stand for the tree it claims to have checked; fail the step
9228        // instead of the patch.
9229        state.event(
9230            "verify",
9231            format!(
9232                "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
9233                worktree.display(),
9234                short(head)
9235            ),
9236        );
9237        guard.release();
9238        return (
9239            vec![CommandOutcome {
9240                command: "(confirming the shared build cache is fresh)".to_owned(),
9241                code: None,
9242                output_tail: e.to_string(),
9243                duration_ms: started.elapsed().as_millis() as u64,
9244                resource_blocked: true,
9245            }],
9246            false,
9247        );
9248    }
9249    let remaining = budget.saturating_sub(started.elapsed());
9250    let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
9251    // A timed-out command's process was only *asked* to die (`kill_on_drop`,
9252    // `start_kill`); confirm it actually has before handing the directory to
9253    // the next acquirer. See `wait_for_timed_out_children_to_die`'s own doc
9254    // for what this can and cannot see.
9255    if !timed_out_pids.is_empty() {
9256        wait_for_timed_out_children_to_die(&timed_out_pids).await;
9257    }
9258    guard.release();
9259    (outcomes, retried)
9260}
9261
9262/// Poll `pids` — commands [`run_commands`] reports as still running when its
9263/// own timeout elapsed — until every one is confirmed gone, or
9264/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
9265///
9266/// Real confirmation where confirmation is possible, not a substitute for
9267/// full process-tree observation: a grandchild the timed-out process spawned
9268/// and that survives independently of it is invisible to a pid check the
9269/// same way it always was, and continuing to observe and collect *that*
9270/// stays a different piece of work with its own owner. This only narrows a
9271/// fixed blind wait into an actual check of the pids this process does know
9272/// about.
9273async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
9274    wait_for_pids_with(
9275        pids,
9276        crate::proc::pid_alive,
9277        LEASE_RELEASE_POLL,
9278        LEASE_RELEASE_MAX_WAIT,
9279    )
9280    .await;
9281}
9282
9283/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
9284/// interval and ceiling supplied by the caller, so the polling *logic* -
9285/// returns as soon as every pid reports dead, gives up at the ceiling
9286/// otherwise - is testable on millisecond durations without asking the real
9287/// OS about a pid at all.
9288async fn wait_for_pids_with<F: Fn(u32) -> bool>(
9289    pids: &[u32],
9290    alive: F,
9291    poll: Duration,
9292    max_wait: Duration,
9293) {
9294    let deadline = Instant::now() + max_wait;
9295    loop {
9296        if pids.iter().all(|&pid| !alive(pid)) {
9297            return;
9298        }
9299        if Instant::now() >= deadline {
9300            return;
9301        }
9302        tokio::time::sleep(poll).await;
9303    }
9304}
9305
9306/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
9307/// admission that it could not even get a verify command to run, as opposed
9308/// to evidence the command actually produced? A caller that would otherwise
9309/// read a resource-blocked outcome as a red command must check this first:
9310/// see [`Runner::gate`], which retries rather than records `Blocked` when
9311/// this is true.
9312fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
9313    outcomes.iter().any(|o| o.resource_blocked)
9314}
9315
9316/// What [`Runner::gate_fix_round`] decided.
9317enum GateFix {
9318    /// The tree changed and `verify.e2e` is still green: run the gate again.
9319    Retry,
9320    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
9321    /// last failure stands and the run ends blocked.
9322    Stop,
9323    /// `verify.e2e` could not run after the fix (magi's own contention):
9324    /// decide nothing now, a later reentry retries.
9325    Defer,
9326}
9327
9328/// Is every red command in `outcomes` an ordinary failure the code could
9329/// explain: it ran, exited non-zero, and said something?
9330///
9331/// A timeout, a spawn failure and a killed process all leave `code` `None`;
9332/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
9333/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
9334/// what the command is stays the gate's business.
9335fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
9336    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
9337    red.peek().is_some()
9338        && red.all(|o| {
9339            !o.resource_blocked
9340                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
9341                && !o.output_tail.trim().is_empty()
9342        })
9343}
9344
9345/// Describe one verify command's outcome for the event log, distinguishing a
9346/// build/link failure — the toolchain never produced a binary to run — from
9347/// an actual test failure, since only the latter is a verdict on the patch.
9348fn e2e_outcome_label(o: &CommandOutcome) -> String {
9349    if o.ok() {
9350        return "pass".to_owned();
9351    }
9352    let reason = if o.build_failed() {
9353        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
9354    } else {
9355        format!("FAIL ({:?})", o.code)
9356    };
9357    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
9358}
9359
9360/// Run `verify.e2e`, retrying once if the first attempt could not build or
9361/// link — a build/link failure is frequently a race against a shared
9362/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
9363/// `verify` event per command, tagged with `context` (normally `"round N"`)
9364/// so the two call sites that need this — the ordinary per-round leg in
9365/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
9366/// it will ever call a round green — read identically in the event log.
9367async fn run_e2e_with_retry(
9368    state: &mut RunState,
9369    shell: &[String],
9370    commands: &[String],
9371    worktree: &Path,
9372    timeout: Duration,
9373    context: &str,
9374) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
9375    let (mut e2e, mut timed_out_pids) = run_commands(
9376        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
9377    )
9378    .await;
9379    for o in &e2e {
9380        state.event(
9381            "verify",
9382            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
9383        );
9384    }
9385    // A build/link failure is not a verdict on the patch — it is frequently a
9386    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
9387    // one retry before letting a red like that decide the round.
9388    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
9389    if verify_retried {
9390        state.event(
9391            "verify",
9392            format!(
9393                "{context}: verify could not build/link, not a test result — retrying once \
9394                 before concluding"
9395            ),
9396        );
9397        let retried = run_commands(
9398            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
9399        )
9400        .await;
9401        e2e = retried.0;
9402        // Both attempts' timeouts matter, not just the last one: the first
9403        // attempt's descendants may still be alive alongside the retry's.
9404        timed_out_pids.extend(retried.1);
9405        for o in &e2e {
9406            state.event(
9407                "verify",
9408                format!(
9409                    "{context}: retry `{}` -> {}",
9410                    o.command,
9411                    e2e_outcome_label(o)
9412                ),
9413            );
9414        }
9415    }
9416    (e2e, verify_retried, timed_out_pids)
9417}
9418
9419/// Run configured shell commands in `cwd`, in order. The second element is
9420/// the pid of every command that hit `timeout` and was still running when
9421/// this stopped waiting on it (best-effort: `None` when the platform did not
9422/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
9423/// that releases a shared resource afterward needs to know.
9424///
9425/// Records `task` into [`RunState::active`] at every command boundary
9426/// (`RunState::task_command`) and clears it once the whole list has run
9427/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
9428/// for minutes with no seat and no output of its own to show for it (see
9429/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
9430/// apart from "not yet run" without this), and this is the only place that
9431/// knows which command is running right now and how many are left. Three
9432/// saves per command — start, not per second — matching the same "only at a
9433/// boundary" rule [`wave`] already follows for seats.
9434#[allow(clippy::too_many_arguments)]
9435async fn run_commands(
9436    state: &mut RunState,
9437    node: &str,
9438    task: &str,
9439    attempt: usize,
9440    shell: &[String],
9441    commands: &[String],
9442    cwd: &Path,
9443    timeout: Duration,
9444) -> (Vec<CommandOutcome>, Vec<u32>) {
9445    if commands.is_empty() {
9446        // Nothing to mark as running and nothing to clear — an empty list
9447        // means "not configured", and touching `active` (or the disk) over
9448        // that would be a write for every round of a repo with no
9449        // `verify.e2e` / `verify.gate` commands at all.
9450        return (Vec::new(), Vec::new());
9451    }
9452    let mut out = Vec::new();
9453    let mut timed_out_pids = Vec::new();
9454    let total = commands.len();
9455    for (idx, command) in commands.iter().enumerate() {
9456        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
9457        if let Err(e) = state.save() {
9458            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
9459        }
9460        let started = Instant::now();
9461        let mut cmd = tokio::process::Command::new(&shell[0]);
9462        cmd.quiet();
9463        cmd.args(&shell[1..])
9464            .arg(command)
9465            .current_dir(cwd)
9466            .stdin(std::process::Stdio::null())
9467            .stdout(std::process::Stdio::piped())
9468            .stderr(std::process::Stdio::piped())
9469            .kill_on_drop(true);
9470        let spawned = cmd.spawn();
9471        let (code, body) = match spawned {
9472            Ok(child) => {
9473                // Captured before the child is consumed below: `kill_on_drop`
9474                // only *asks* the process to die when the timeout branch
9475                // drops it, and the pid is the only way anyone downstream can
9476                // later check whether that request actually took.
9477                let pid = child.id();
9478                match tokio::time::timeout(timeout, child.wait_with_output()).await {
9479                    Ok(Ok(o)) => {
9480                        let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
9481                        body.push_str(&String::from_utf8_lossy(&o.stderr));
9482                        (o.status.code(), body)
9483                    }
9484                    Ok(Err(e)) => (None, format!("failed to run: {e}")),
9485                    Err(_) => {
9486                        if let Some(pid) = pid {
9487                            timed_out_pids.push(pid);
9488                        }
9489                        (None, format!("timed out after {}s", timeout.as_secs()))
9490                    }
9491                }
9492            }
9493            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
9494        };
9495        out.push(CommandOutcome {
9496            command: command.clone(),
9497            code,
9498            output_tail: tail(&body, OUTPUT_TAIL),
9499            duration_ms: started.elapsed().as_millis() as u64,
9500            resource_blocked: false,
9501        });
9502    }
9503    state.task_finished(task);
9504    if let Err(e) = state.save() {
9505        tracing::warn!("could not persist the end of {task}: {e:#}");
9506    }
9507    (out, timed_out_pids)
9508}
9509
9510/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
9511/// section (`report::run`) and in the `merge` event this node records — for
9512/// the operator to run by hand.
9513///
9514/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
9515/// branch whose ruleset forbids merge commits (GitHub's "must not contain
9516/// merge commits", or "require linear history") rejects the push a `--no-ff`
9517/// merge would produce, which is exactly the guidance this function replaces.
9518/// `message`'s first line becomes the squash commit's subject, matching the
9519/// note `report::run` prints alongside this command — see that function for
9520/// why an explicit subject is not optional there.
9521fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
9522    let repo = repo.display();
9523    match style {
9524        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
9525        MergeStyle::Squash => {
9526            // The subject sits inside double quotes, and a title an agent
9527            // wrote may carry the characters that break out of them.
9528            let subject = message
9529                .lines()
9530                .next()
9531                .unwrap_or(branch)
9532                .replace(['\\', '"', '$', '`'], "");
9533            format!(
9534                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
9535            )
9536        }
9537        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
9538    }
9539}
9540
9541/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
9542/// under the hood, rejects a `title` over 256 characters and the whole
9543/// command fails — no PR at all, for a run whose body was otherwise fine
9544/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
9545/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
9546/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
9547/// this bound. It is a margin, not a guarantee — a title packed
9548/// with multi-unit characters could still in principle land close to the
9549/// edge, but a real task title's occasional emoji or accented letter fits
9550/// comfortably inside it.
9551const PR_TITLE_MAX: usize = 240;
9552
9553/// A pull request title from the opening line of `text`, or `None` when that
9554/// line is not English (GitHub text is) or has no letters.
9555///
9556/// A line within `max` is kept as is. A longer one is cut at the end of its
9557/// first sentence when that falls inside `max`, else at a word boundary with a
9558/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
9559/// make every merely-truncated title look non-English). The language check
9560/// runs on the kept text before any mark is added, so only what GitHub will
9561/// show is judged: an English opening followed by non-ASCII far past the cut
9562/// still passes.
9563fn english_title(text: &str, max: usize) -> Option<String> {
9564    let line = queue::first_line(text)?;
9565    let chars: Vec<char> = line.chars().collect();
9566    let (kept, mark) = if chars.len() <= max {
9567        (line.to_owned(), "")
9568    } else if let Some(end) = sentence_end(&chars, max) {
9569        (chars[..end].iter().collect::<String>(), "")
9570    } else {
9571        let room = max.saturating_sub(3);
9572        // Cut at the last space inside the room; when the char just past the
9573        // room is a space the room already ends on a word.
9574        let cut = if chars[room].is_whitespace() {
9575            room
9576        } else {
9577            chars[..room]
9578                .iter()
9579                .rposition(|c| c.is_whitespace())
9580                .unwrap_or(room)
9581        };
9582        let head: String = chars[..cut].iter().collect();
9583        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
9584        (head.to_owned(), "...")
9585    };
9586    if kept.is_empty() || !kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic()) {
9587        return None;
9588    }
9589    Some(format!("{kept}{mark}"))
9590}
9591
9592/// The char length of the first sentence of `chars` when it ends within `max`
9593/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
9594/// abbreviations so `e.g. foo` does not end a title early.
9595fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
9596    const MIN: usize = 20;
9597    for i in MIN..max.min(chars.len()) {
9598        if !matches!(chars[i], '.' | '!' | '?') {
9599            continue;
9600        }
9601        let Some(&next) = chars.get(i + 1) else {
9602            continue;
9603        };
9604        if !next.is_whitespace() {
9605            continue;
9606        }
9607        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
9608        if after.is_some_and(|c| c.is_ascii_lowercase()) {
9609            continue;
9610        }
9611        let word: String = chars[..i]
9612            .iter()
9613            .rev()
9614            .take_while(|c| !c.is_whitespace())
9615            .collect::<Vec<_>>()
9616            .into_iter()
9617            .rev()
9618            .collect();
9619        let word = word.to_ascii_lowercase();
9620        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
9621            continue;
9622        }
9623        let end = chars[..i]
9624            .iter()
9625            .rposition(|c| !c.is_whitespace())
9626            .map_or(i, |p| p + 1);
9627        return Some(end);
9628    }
9629    None
9630}
9631
9632/// What `merge = "pr"` (and the merge commit of the other modes) says about a
9633/// change: a title and a body describing what was *implemented*, not the task
9634/// that asked for it. A task reads as a request; a reader of the merged
9635/// history wants the change.
9636struct PrMessage {
9637    title: String,
9638    body: String,
9639}
9640
9641impl PrMessage {
9642    /// Title, blank line, body. The first line is the squash/merge commit
9643    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
9644    /// has to stay one sensible line.
9645    fn commit_message(&self) -> String {
9646        format!("{}\n\n{}", self.title, self.body)
9647    }
9648}
9649
9650/// The text after a leading `TITLE:` (any case) on `line`.
9651fn title_marker(line: &str) -> Option<&str> {
9652    let line = line.trim();
9653    let head = line.get(..6)?;
9654    head.eq_ignore_ascii_case("title:")
9655        .then(|| line[6..].trim())
9656}
9657
9658/// The implementer's own one-line title: the `TITLE:` line the implement
9659/// prompt asks for at the top of its SUMMARY. Candidate commits are all
9660/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
9661/// source, and a title that says as much is refused here too.
9662fn summary_title(summary: &str) -> Option<String> {
9663    let first = summary.lines().find(|l| !l.trim().is_empty())?;
9664    let raw = title_marker(first)?;
9665    if raw.is_empty() {
9666        return None;
9667    }
9668    let title = queue::title_from(raw, PR_TITLE_MAX);
9669    let lower = title.to_ascii_lowercase();
9670    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
9671        return None;
9672    }
9673    Some(title)
9674}
9675
9676/// How `open_review`'s instruction begins; see [`landing_title`].
9677const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
9678
9679/// Marker `open_review` gives a candidate that nothing in the roster wrote.
9680const EXISTING_BRANCH: &str = "(existing branch)";
9681
9682/// Does this run review work that already existed, rather than implement a
9683/// task? Runs recorded before `reviewed_commits` existed carry only the
9684/// candidate marker.
9685fn is_review_run(state: &RunState) -> bool {
9686    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
9687}
9688
9689/// The title of a review-only run: the subject of the oldest commit under
9690/// review. Later commits are usually fixups, and `instruction` is the review
9691/// prompt, which says nothing about the change. GitHub text is English, so a
9692/// non-ASCII or blank subject yields `None` and the caller's neutral title.
9693fn review_title(state: &RunState) -> Option<String> {
9694    english_subject(state.reviewed_commits.as_ref()?.first()?)
9695}
9696
9697/// `raw` as a pull request title, or `None` when it is blank, not English
9698/// (GitHub text is), or one of magi's own candidate commit subjects.
9699fn english_subject(raw: &str) -> Option<String> {
9700    let raw = raw.trim();
9701    if raw.is_empty() || !raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic()) {
9702        return None;
9703    }
9704    let title = queue::title_from(raw, PR_TITLE_MAX);
9705    let lower = title.to_ascii_lowercase();
9706    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
9707        return None;
9708    }
9709    Some(title)
9710}
9711
9712/// What a review-only run's branch says about itself, read at the moment the
9713/// pull request is opened.
9714#[derive(Debug, Clone, PartialEq, Eq)]
9715struct BranchFacts {
9716    /// `(subject, body)` of each commit, oldest first.
9717    commits: Vec<(String, String)>,
9718    /// Trimmed `git diff --stat`.
9719    stat: String,
9720}
9721
9722/// Longest diff stat shown: this many file lines plus the summary line.
9723const STAT_FILE_LINES: usize = 25;
9724/// Cap on the commit list, well inside GitHub's 65536-character body limit.
9725const COMMITS_MAX_CHARS: usize = 20_000;
9726
9727/// Read the commits and diff stat of `base..branch`. `None` when git cannot
9728/// say or finds nothing, so the caller falls back to what the run recorded.
9729async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
9730    let commits = git::commit_log(repo, base, branch).await.ok()?;
9731    if commits.is_empty() {
9732        return None;
9733    }
9734    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
9735    let lines: Vec<&str> = stat.lines().collect();
9736    let stat = if lines.len() > STAT_FILE_LINES + 1 {
9737        let omitted = lines.len() - 1 - STAT_FILE_LINES;
9738        let more = format!(" ... {omitted} more file(s)");
9739        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
9740        kept.push(&more);
9741        kept.push(lines[lines.len() - 1]);
9742        kept.join("\n")
9743    } else {
9744        lines.join("\n")
9745    };
9746    Some(BranchFacts { commits, stat })
9747}
9748
9749/// Defang what would break the surrounding markdown: a closing `</details>`
9750/// and a code fence.
9751fn markdown_safe(text: &str) -> String {
9752    text.replace("</details>", "&lt;/details&gt;")
9753        .replace("\x60\x60\x60", "~~~")
9754}
9755
9756fn neutral_title(state: &RunState, winner: char) -> String {
9757    format!(
9758        "chore: land candidate {} of run {}",
9759        winner.to_ascii_uppercase(),
9760        state.id
9761    )
9762}
9763
9764/// The pull request title to hand to `land::merge_subject`. A review-only run
9765/// opened by an earlier build titled its pull request with the review prompt;
9766/// that title is dropped (empty, so the fallback applies) rather than landed.
9767/// Any other title, including an operator's rename, passes through untouched,
9768/// and so does every title of a run that implements a task.
9769pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
9770    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
9771        ""
9772    } else {
9773        pr_title
9774    }
9775}
9776
9777/// What the squash subject falls back to when the pull request title is empty
9778/// or candidate-shaped: for a review-only run the derived title, never the
9779/// review prompt held in `instruction`.
9780pub fn landing_subject_source(state: &RunState) -> String {
9781    if is_review_run(state) {
9782        let winner = state.candidates.first().map_or('A', |c| c.label);
9783        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
9784    }
9785    state.instruction.clone()
9786}
9787
9788/// `summary` without its `TITLE:` line, which the pull request title already
9789/// carries.
9790fn summary_without_title(summary: &str) -> String {
9791    let mut lines = summary.trim().lines().peekable();
9792    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
9793        lines.next();
9794    }
9795    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
9796}
9797
9798/// The pull request title and body for the winning candidate.
9799///
9800/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
9801/// the task's own opening line via [`queue::title_from`] when there is none.
9802/// `state.instruction` can open with blank lines (`task_text` only rejects a
9803/// body that is blank *entirely*), which `title_from` skips.
9804///
9805/// Body: the implementer's summary and the fixer's notes, then — when the
9806/// winning review round was not clean — the findings still open and whatever
9807/// the fixer declined, so `merge = "pr"` hands the reader the same material
9808/// `magi show` does. The task follows inside a collapsed block, and the
9809/// footer repeats the run and candidate as plain tags for a reader holding
9810/// only the merged commit or the PR body.
9811#[cfg(test)]
9812fn pr_message(state: &RunState, winner: char) -> PrMessage {
9813    pr_message_with(state, winner, None)
9814}
9815
9816/// [`pr_message`] with what the branch of a review-only run says about itself.
9817/// `facts` is ignored for a run that implements a task.
9818#[cfg(test)]
9819fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
9820    let raw = pr_message_raw(
9821        state,
9822        winner,
9823        facts,
9824        crate::pr_outcome::FinalHead::Unreported,
9825    );
9826    let id = crate::scrub::Identity::current();
9827    PrMessage {
9828        title: crate::scrub::scrub(&raw.title, &id),
9829        body: crate::scrub::scrub(&raw.body, &id),
9830    }
9831}
9832
9833fn pr_message_raw(
9834    state: &RunState,
9835    winner: char,
9836    facts: Option<&BranchFacts>,
9837    head: crate::pr_outcome::FinalHead<'_>,
9838) -> PrMessage {
9839    let summary = state
9840        .candidates
9841        .iter()
9842        .find(|c| c.label == winner)
9843        .map(|c| c.summary.as_str())
9844        .unwrap_or_default();
9845    // The fallback is the operator's own words and may not be English; GitHub
9846    // text always is, so a non-English task gets a neutral title instead.
9847    let review = is_review_run(state);
9848    let title = if review {
9849        facts
9850            .and_then(|f| english_subject(&f.commits.first()?.0))
9851            .or_else(|| review_title(state))
9852            .or_else(|| {
9853                state
9854                    .candidates
9855                    .iter()
9856                    .find(|c| c.label == winner)
9857                    .filter(|c| !c.branch.starts_with("magi/"))
9858                    .and_then(|c| english_subject(&c.branch))
9859            })
9860            .unwrap_or_else(|| neutral_title(state, winner))
9861    } else {
9862        summary_title(summary).unwrap_or_else(|| {
9863            english_title(&state.instruction, PR_TITLE_MAX)
9864                .unwrap_or_else(|| neutral_title(state, winner))
9865        })
9866    };
9867
9868    let mut body = String::new();
9869    let what = summary_without_title(summary);
9870    if !what.is_empty() {
9871        body.push_str("## Summary\n\n");
9872        body.push_str(&what);
9873        body.push_str("\n\n");
9874    }
9875
9876    // The last round is usually a clean verification pass with no fix of its
9877    // own, so every round's notes are read, not just the final one's. Sentences
9878    // the final state contradicts are left out (see `pr_outcome`).
9879    body.push_str(&crate::pr_outcome::review_fixes(state, head).0);
9880    body.push_str(&crate::pr_outcome::final_verification(state, head));
9881    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
9882
9883    let open = state.open_findings();
9884    if !open.is_empty() {
9885        body.push_str("## Open review findings\n\n");
9886        for f in &open {
9887            body.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
9888        }
9889        body.push('\n');
9890    }
9891
9892    if let Some(fix) = fix
9893        && !fix.rejected.is_empty()
9894    {
9895        body.push_str("## Declined by the fixer\n\n");
9896        for r in &fix.rejected {
9897            body.push_str(&format!("- `{}`: {}\n", r.id, r.why));
9898        }
9899        body.push('\n');
9900    }
9901
9902    if review {
9903        // The review prompt is not the task; list what the branch carries.
9904        body.push_str("## Commits under review\n\n");
9905        if let Some(facts) = facts {
9906            let mut left = COMMITS_MAX_CHARS;
9907            for (i, (subject, text)) in facts.commits.iter().enumerate() {
9908                let mut entry = format!("- {}\n", markdown_safe(subject));
9909                for l in markdown_safe(text).lines() {
9910                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
9911                }
9912                if left == 0 {
9913                    body.push_str(&format!(
9914                        "- ... {} more commit(s)\n",
9915                        facts.commits.len() - i
9916                    ));
9917                    break;
9918                }
9919                if entry.len() > left {
9920                    // Even the first commit is cut: one huge body must not
9921                    // push the whole description past GitHub's limit.
9922                    let mut end = left;
9923                    while !entry.is_char_boundary(end) {
9924                        end -= 1;
9925                    }
9926                    entry.truncate(end);
9927                    entry.push_str("\n  ... (truncated)\n");
9928                    left = 0;
9929                } else {
9930                    left -= entry.len();
9931                }
9932                body.push_str(&entry);
9933            }
9934            if !facts.stat.trim().is_empty() {
9935                body.push_str(&format!(
9936                    "\n## Diff stat\n\n```\n{}\n```\n",
9937                    markdown_safe(facts.stat.trim())
9938                ));
9939            }
9940        } else {
9941            match &state.reviewed_commits {
9942                Some(subjects) => {
9943                    for s in subjects {
9944                        body.push_str(&format!("- {}\n", s.trim()));
9945                    }
9946                }
9947                None => {
9948                    // An older run kept only the prompt, with the commit list
9949                    // after its first paragraph.
9950                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
9951                    body.push_str(rest.trim());
9952                    body.push('\n');
9953                }
9954            }
9955        }
9956    } else {
9957        let task = state.instruction.trim();
9958        let task = if task.is_empty() {
9959            "(empty task)"
9960        } else {
9961            task
9962        };
9963        body.push_str(&format!(
9964            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
9965            task.replace("</details>", "&lt;/details&gt;")
9966        ));
9967    }
9968
9969    body.push_str(&format!(
9970        "\n---\nmagi:run/{} magi:candidate-{}\n",
9971        state.id,
9972        winner.to_ascii_lowercase()
9973    ));
9974
9975    PrMessage { title, body }
9976}
9977
9978/// The task with what the repository says about the existing work it names
9979/// appended, so an implementer knows what it started from and what it must
9980/// not redo. Unchanged when the task names nothing.
9981fn seeded_instruction(state: &RunState) -> String {
9982    match refs::describe(&state.seeds) {
9983        Some(facts) => format!(
9984            "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
9985             Candidates start from the unmerged branch named above, when there \
9986             is one, and carry any unmerged commit named by sha as a \
9987             cherry-pick. Check that this is what the task meant before \
9988             building on it.",
9989            state.instruction
9990        ),
9991        None => state.instruction.clone(),
9992    }
9993}
9994
9995/// Does the winner have no commits ahead of the base it would land on?
9996/// Any failure to find out reads as "not empty": the merge then behaves as it
9997/// always did rather than refusing on a guess.
9998async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
9999    let base = &state.base_branch;
10000    let mut against = base.clone();
10001    if mode == MergeMode::Pr {
10002        // A pull request lands on the remote's base, never the local branch
10003        // of the same name: if that cannot be read, "not empty" is the safe
10004        // answer.
10005        let remote = &state.config.merge.remote;
10006        let tracking = format!("{remote}/{base}");
10007        let fetched = git::fetch(repo, remote, base).await;
10008        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
10009            against = tracking;
10010        } else {
10011            return false;
10012        }
10013    }
10014    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
10015}
10016
10017/// Why nothing was opened for an empty winner, with what the task's own
10018/// references resolved to.
10019fn empty_candidate_detail(state: &RunState, base: &str) -> String {
10020    let mut detail = format!(
10021        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
10022         nothing to open a pull request for"
10023    );
10024    match refs::describe(&state.seeds) {
10025        Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
10026        None => detail.push_str(
10027            "\nThe task names no existing branch or commit; if it means to land work \
10028             that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
10029        ),
10030    }
10031    detail
10032}
10033
10034/// What the `Pr` merge does once it knows whether the branch already has an
10035/// open pull request.
10036#[derive(Debug, PartialEq, Eq)]
10037enum PrPlan {
10038    Create,
10039    Adopt { url: String, title: String },
10040    Stop(String),
10041}
10042
10043/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
10044/// a failed lookup -> stop with the real reason. Never guesses.
10045fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
10046    match found {
10047        Ok(land::OpenPr::None) => PrPlan::Create,
10048        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
10049        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
10050            "several open pull requests exist for this branch, not picking one: {}",
10051            urls.join(" ")
10052        )),
10053        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
10054    }
10055}
10056
10057/// `gh pr create`, returning the PR url.
10058pub(crate) async fn gh_pr_create(
10059    cwd: &Path,
10060    base: &str,
10061    head: &str,
10062    title: &str,
10063    body: &str,
10064) -> Result<String> {
10065    let out = tokio::process::Command::new("gh")
10066        .args([
10067            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
10068        ])
10069        .current_dir(cwd)
10070        .quiet()
10071        .stdin(std::process::Stdio::null())
10072        .output()
10073        .await
10074        .context("spawn gh")?;
10075    if out.status.success() {
10076        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
10077    } else {
10078        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
10079    }
10080}
10081
10082/// Tear a run's worktrees and branches down.
10083///
10084/// `home` is where the updated `run.json` is saved (via
10085/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
10086/// a housekeeping pass already has its own honest `home` handed to it, and
10087/// falling through to the global here would write back through whichever
10088/// directory some other process or test pinned into that `OnceLock` first,
10089/// not the one the caller actually resolved its `runs` and `state` from.
10090pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
10091    let repo = state.repo.clone();
10092    let root = state.worktree_root();
10093    let winner = state.tally.as_ref().map(|t| t.winner);
10094    let mut removed = Vec::new();
10095
10096    for i in 0..state.candidates.len() {
10097        let c = state.candidates[i].clone();
10098        let is_winner = Some(c.label) == winner;
10099        if is_winner && !drop_winner {
10100            continue;
10101        }
10102        if c.worktree.exists() {
10103            git::worktree_remove(&repo, &c.worktree).await.ok();
10104            removed.push(c.worktree.to_string_lossy().into_owned());
10105        }
10106        // A branch handed to a later run (and its pull request) is not this
10107        // run's to delete.
10108        let handed_over = state.released_branches.contains(&c.branch);
10109        if !handed_over && git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
10110            git::branch_delete(&repo, &c.branch).await.ok();
10111            removed.push(c.branch.clone());
10112        }
10113        state.candidates[i].folded = true;
10114    }
10115
10116    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
10117        let path = name.path();
10118        let keep = !drop_winner
10119            && winner.is_some_and(|w| {
10120                path.file_name()
10121                    .is_some_and(|n| n == format!("cand-{w}").as_str())
10122            });
10123        if keep {
10124            continue;
10125        }
10126        git::worktree_remove(&repo, &path).await.ok();
10127        removed.push(path.to_string_lossy().into_owned());
10128    }
10129
10130    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
10131    // judge worktrees, so once the loop above has cleared all of them out,
10132    // the parent is a bare directory nobody else was ever going to remove -
10133    // git only ever managed what was inside it. Left alone, one of these
10134    // accumulates per fully-folded run; the operator's own machine had 74.
10135    // `remove_if_empty` re-checks rather than assuming: a run whose winner
10136    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
10137    // so does anything a run never claimed that happens to share the bay.
10138    remove_if_empty(&root);
10139
10140    if state.enabled_worktree_config && drop_winner {
10141        // A release, not a raw disable: some sibling run in this repository
10142        // may still hold its own reference (see `git::acquire_worktree_config`),
10143        // and only the last release actually turns the setting back off.
10144        git::release_worktree_config(&repo).await.ok();
10145        state.enabled_worktree_config = false;
10146    }
10147    state.save_under(home)?;
10148    Ok(removed)
10149}
10150
10151/// Remove `dir` if it exists and has nothing in it.
10152///
10153/// Best-effort and silent by design: a directory that is not empty (a run
10154/// whose winner is still parked there, a stray file some other process left)
10155/// is exactly the case this must refuse, and a directory that is already gone
10156/// is not a failure worth reporting either. `std::fs::remove_dir` itself
10157/// already refuses a non-empty directory, so the emptiness check below is
10158/// belt, not suspenders - it is what keeps this from ever attempting the
10159/// removal in the case that matters, rather than trusting `remove_dir`'s
10160/// error path to have no side effects if it ever changed.
10161fn remove_if_empty(dir: &Path) {
10162    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
10163        std::fs::remove_dir(dir).ok();
10164    }
10165}
10166
10167/// Severity of the worst open finding in the last review round, for reporting.
10168pub fn worst_open(state: &RunState) -> Option<Severity> {
10169    state
10170        .reviews
10171        .last()?
10172        .reviews
10173        .iter()
10174        .flat_map(|r| r.findings.iter())
10175        .map(|f| f.severity)
10176        .max()
10177}
10178
10179/// `[worktree] setup` hides its products per worktree, which needs
10180/// `extensions.worktreeConfig`. Taken once per run and released with the run's
10181/// worktrees, like the hook's; every place that runs setup calls this first.
10182async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
10183    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
10184        git::acquire_worktree_config(repo).await?;
10185        state.enabled_worktree_config = true;
10186    }
10187    Ok(())
10188}
10189
10190#[cfg(test)]
10191mod tests {
10192    #[test]
10193    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
10194        let leaked = vec!["chore(deps): update a crate".to_owned()];
10195        let own = "fix(daemon): apply a chosen action";
10196        assert!(should_retitle("", own, &leaked));
10197        assert!(should_retitle(
10198            &format!("{REVIEW_PROMPT_OPENING} `x`"),
10199            own,
10200            &leaked
10201        ));
10202        assert!(should_retitle(
10203            "chore: land candidate A of run 1",
10204            own,
10205            &leaked
10206        ));
10207        assert!(should_retitle(
10208            "magi: candidate A (uncommitted work)",
10209            own,
10210            &leaked
10211        ));
10212        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
10213        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
10214        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
10215    }
10216
10217    #[test]
10218    fn pr_merge_plan_creates_adopts_or_stops() {
10219        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
10220        assert_eq!(
10221            pr_merge_plan(Ok(land::OpenPr::One {
10222                url: "u".into(),
10223                title: "t".into()
10224            })),
10225            PrPlan::Adopt {
10226                url: "u".into(),
10227                title: "t".into()
10228            }
10229        );
10230        let PrPlan::Stop(many) =
10231            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
10232        else {
10233            panic!("many must stop");
10234        };
10235        assert!(many.contains('a') && many.contains('b'));
10236        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
10237            panic!("a failed lookup must stop");
10238        };
10239        assert!(err.contains("bad token"));
10240    }
10241
10242    use super::*;
10243    use crate::run::GateStatus;
10244    use std::collections::BTreeMap;
10245    use std::time::Duration;
10246
10247    fn conductor() -> AgentSpec {
10248        AgentSpec {
10249            id: "conductor".to_owned(),
10250            kind: crate::config::AgentKind::Command,
10251            model: None,
10252            command: vec!["true".to_owned()],
10253            extra_args: Vec::new(),
10254            env: BTreeMap::new(),
10255            prompt_delivery: None,
10256        }
10257    }
10258
10259    fn spec(id: &str) -> AgentSpec {
10260        AgentSpec {
10261            id: id.to_owned(),
10262            kind: crate::config::AgentKind::Command,
10263            model: None,
10264            command: vec!["true".to_owned()],
10265            extra_args: Vec::new(),
10266            env: BTreeMap::new(),
10267            prompt_delivery: None,
10268        }
10269    }
10270
10271    fn ids(xs: &[&str]) -> BTreeSet<String> {
10272        xs.iter().map(|s| (*s).to_owned()).collect()
10273    }
10274
10275    #[test]
10276    fn pick_successor_skips_an_agent_another_seat_holds() {
10277        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
10278        let roster = [spec("a"), spec("b"), spec("c")];
10279        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
10280        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
10281    }
10282
10283    #[test]
10284    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
10285        // The other seat started on c but was handed to d; c is free again.
10286        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
10287        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
10288        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
10289        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
10290        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
10291    }
10292
10293    #[test]
10294    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
10295        let roster = [spec("a"), spec("b")];
10296        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
10297        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
10298        let carried = ids(&["b"]);
10299        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
10300        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
10301    }
10302
10303    #[test]
10304    fn pick_successor_returns_none_without_any_untried_successor() {
10305        let roster = [spec("a"), spec("b")];
10306        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
10307        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
10308        let carried = ids(&["a"]);
10309        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
10310    }
10311
10312    #[test]
10313    fn pick_successor_rescue_avoids_another_seats_occupant() {
10314        // b is a carried failure and free; a is held by the other seat.
10315        let roster = [spec("a"), spec("b"), spec("c")];
10316        let carried = ids(&["b", "c"]);
10317        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
10318        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
10319    }
10320
10321    #[test]
10322    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
10323        let roster = [spec("a"), spec("b"), spec("c")];
10324        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
10325        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
10326    }
10327
10328    #[test]
10329    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
10330        let roster = [spec("a"), spec("b"), spec("c")];
10331        let failed = ids(&["a", "b", "c"]);
10332        // Rescue looks at the whole roster, once per id, then runs out.
10333        let mut tried = ids(&["b"]);
10334        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
10335        assert_eq!(first.id, "a");
10336        tried.insert(first.id.clone());
10337        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
10338        assert_eq!(second.id, "c");
10339        tried.insert(second.id.clone());
10340        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
10341    }
10342
10343    #[test]
10344    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
10345        let roster = [spec("a"), spec("b")];
10346        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
10347        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
10348    }
10349
10350    #[test]
10351    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
10352        let roster = [spec("a"), spec("b"), spec("c")];
10353        let h = SeatHistory {
10354            failed: ids(&["a"]),
10355            last_ok: Some("b".to_owned()),
10356            last_fail: None,
10357        };
10358        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
10359        // A last answerer that left the roster, or later failed, is ignored.
10360        let gone = SeatHistory {
10361            last_ok: Some("zzz".to_owned()),
10362            ..h.clone()
10363        };
10364        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
10365        let failed = SeatHistory {
10366            failed: ids(&["a", "b"]),
10367            last_ok: Some("b".to_owned()),
10368            last_fail: None,
10369        };
10370        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
10371    }
10372
10373    #[test]
10374    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
10375        let first = SeatState::new("review-1", "alpha", 7);
10376        let next = handover_seat("review-1", "gamma", 7);
10377        assert_ne!(first.claude_session, next.claude_session);
10378    }
10379
10380    #[test]
10381    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
10382        let mut state = state_with_summary("x", "y");
10383        let a = handover_seat("review-1", "beta", state.next_seat_seed());
10384        let b = handover_seat("review-1", "beta", state.next_seat_seed());
10385        assert_ne!(a.claude_session, b.claude_session);
10386    }
10387
10388    #[test]
10389    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
10390        let roster = [spec("a"), spec("b")];
10391        let h = SeatHistory {
10392            failed: ids(&["a", "b"]),
10393            ..SeatHistory::default()
10394        };
10395        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
10396        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
10397        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
10398    }
10399
10400    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
10401    // fallback loop depends on to terminate: it must walk forward from the
10402    // seat's own position, never restart at the front of the roster, and it
10403    // must never hand back an id already tried, however many times that id
10404    // happens to appear.
10405
10406    #[test]
10407    fn failure_signature_ignores_numbers_and_paths() {
10408        assert_eq!(
10409            failure_signature("exited with Some(2) and no usable output"),
10410            failure_signature("exited with Some(137) and no usable output")
10411        );
10412        assert_eq!(
10413            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
10414            failure_signature("cannot open /var/x.txt: denied")
10415        );
10416        assert_ne!(failure_signature("boom"), failure_signature("bang"));
10417    }
10418
10419    #[test]
10420    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
10421        let other = FailClass::Other("x".into());
10422        assert!(should_hand_over(None, &FailClass::Quota));
10423        assert!(should_hand_over(Some(&other), &FailClass::Quota));
10424        assert!(should_hand_over(
10425            Some(&FailClass::Timeout),
10426            &FailClass::Timeout
10427        ));
10428        assert!(should_hand_over(None, &other));
10429        assert!(!should_hand_over(Some(&other), &other));
10430        assert!(should_hand_over(
10431            Some(&other),
10432            &FailClass::Other("y".into())
10433        ));
10434        // A quota or timeout in between ends the run of identical failures.
10435        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
10436        assert!(should_hand_over(Some(&FailClass::Quota), &other));
10437    }
10438
10439    #[test]
10440    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
10441        let a = SeatState::new("judge-1", "alpha", 7);
10442        let b = handover_seat("judge-1", "beta", 7);
10443        assert_ne!(a.claude_session, b.claude_session);
10444        assert_eq!(b.turns, 0);
10445    }
10446
10447    #[test]
10448    fn a_timeout_is_classified_apart_from_other_failures() {
10449        assert_eq!(
10450            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
10451            Some(FailClass::Timeout)
10452        );
10453        assert!(matches!(
10454            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
10455            Some(FailClass::Other(_))
10456        ));
10457    }
10458
10459    #[test]
10460    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
10461        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
10462        let tried = BTreeSet::from(["beta".to_owned()]);
10463        // beta sits at index 1; the next candidate is gamma, never alpha —
10464        // which is very likely a different candidate slot's own agent.
10465        let next = next_untried_in_roster(&roster, 1, &tried);
10466        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
10467    }
10468
10469    #[test]
10470    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
10471        let roster = vec![spec("alpha"), spec("beta")];
10472        let tried = BTreeSet::from(["beta".to_owned()]);
10473        // beta is the roster's last entry: nothing follows it, and alpha —
10474        // earlier in the roster, almost certainly a different candidate
10475        // slot's own agent — must not be reached by wrapping back to it.
10476        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
10477    }
10478
10479    #[test]
10480    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
10481        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
10482        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
10483        // beta (index 1) and gamma (index 2, the only entry after it) have
10484        // both been tried; alpha (index 0) never has, but it comes before
10485        // beta's own position, so there is nothing further for this seat.
10486        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
10487    }
10488
10489    #[test]
10490    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
10491        let roster = vec![spec("a"), spec("a"), spec("b")];
10492        let tried = BTreeSet::from(["a".to_owned()]);
10493        let next = next_untried_in_roster(&roster, 0, &tried);
10494        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
10495    }
10496
10497    #[test]
10498    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
10499        let roster = vec![spec("a"), spec("b")];
10500        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
10501        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
10502    }
10503
10504    #[test]
10505    fn remove_if_empty_only_ever_takes_a_bare_directory() {
10506        let dir = tempfile::tempdir().unwrap();
10507        let bay = dir.path().join("ffff");
10508
10509        // Not there yet: nothing to do, nothing to panic on.
10510        remove_if_empty(&bay);
10511        assert!(!bay.exists());
10512
10513        // Something still inside - the winner's worktree, or a stray file -
10514        // keeps the directory standing.
10515        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
10516        remove_if_empty(&bay);
10517        assert!(bay.exists(), "non-empty directory must survive");
10518
10519        // Once the last entry is gone, so is the directory itself.
10520        std::fs::remove_dir(bay.join("cand-A")).unwrap();
10521        remove_if_empty(&bay);
10522        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
10523    }
10524
10525    // `round_is_clean` is the exact decision this task fixed: a round with a
10526    // seat that never answered must not read the same as a round every seat
10527    // actually reviewed. These are deterministic and process-free by design —
10528    // the equivalent end-to-end check (a real reviewer timing out under a
10529    // live graph run) is a genuine race against wall-clock contention, and a
10530    // spawn slow enough to blow even a generous budget under a loaded test
10531    // run must not turn this specific regression check flaky.
10532
10533    #[test]
10534    fn a_full_panel_that_found_nothing_is_clean() {
10535        assert!(round_is_clean(
10536            0,
10537            true,
10538            2,
10539            2,
10540            0,
10541            IncompleteReviewPolicy::Block
10542        ));
10543    }
10544
10545    #[test]
10546    fn a_missing_seat_is_never_clean_under_the_default_policy() {
10547        assert!(!round_is_clean(
10548            0,
10549            true,
10550            1,
10551            2,
10552            0,
10553            IncompleteReviewPolicy::Block
10554        ));
10555    }
10556
10557    #[test]
10558    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
10559        assert!(!round_is_clean(
10560            1,
10561            true,
10562            1,
10563            2,
10564            0,
10565            IncompleteReviewPolicy::Warn
10566        ));
10567    }
10568
10569    #[test]
10570    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
10571        assert!(round_is_clean(
10572            0,
10573            true,
10574            1,
10575            2,
10576            0,
10577            IncompleteReviewPolicy::Warn
10578        ));
10579    }
10580
10581    #[test]
10582    fn a_full_panel_with_an_open_finding_is_not_clean() {
10583        assert!(!round_is_clean(
10584            1,
10585            true,
10586            2,
10587            2,
10588            0,
10589            IncompleteReviewPolicy::Block
10590        ));
10591    }
10592
10593    #[test]
10594    fn a_full_panel_with_a_red_e2e_is_not_clean() {
10595        assert!(!round_is_clean(
10596            0,
10597            false,
10598            2,
10599            2,
10600            0,
10601            IncompleteReviewPolicy::Block
10602        ));
10603    }
10604
10605    // The stall this task closes: under the default `block` policy, a seat
10606    // missing only because it was rate limited must not force a wait for a
10607    // session limit that will not lift by the next round. `round_is_clean`
10608    // is where that quorum carve-out lives; the review loop around it never
10609    // changes what a reviewer's vote or a finding's severity means.
10610
10611    #[test]
10612    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
10613        // 1 of 2 answered, and the one missing was quota'd — the exact
10614        // "review-2 rate limited (quota)" shape from the field report.
10615        assert!(round_is_clean(
10616            0,
10617            true,
10618            1,
10619            2,
10620            1,
10621            IncompleteReviewPolicy::Block
10622        ));
10623    }
10624
10625    #[test]
10626    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
10627        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
10628        // not a quota loss (`quota_missing` stays 0) — worth another try.
10629        assert!(!round_is_clean(
10630            0,
10631            true,
10632            1,
10633            2,
10634            0,
10635            IncompleteReviewPolicy::Block
10636        ));
10637    }
10638
10639    #[test]
10640    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
10641        assert!(!round_is_clean(
10642            1,
10643            true,
10644            1,
10645            2,
10646            1,
10647            IncompleteReviewPolicy::Block
10648        ));
10649        assert!(!round_is_clean(
10650            0,
10651            false,
10652            1,
10653            2,
10654            1,
10655            IncompleteReviewPolicy::Block
10656        ));
10657    }
10658
10659    #[test]
10660    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
10661        // Every seat quota'd, nobody answered: there is no panel to decide
10662        // on, so this must fall through to the existing block-and-retry
10663        // fallback rather than call an unreviewed patch clean.
10664        assert!(!round_is_clean(
10665            0,
10666            true,
10667            0,
10668            2,
10669            2,
10670            IncompleteReviewPolicy::Block
10671        ));
10672    }
10673
10674    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
10675        CommandOutcome {
10676            command: "test".to_owned(),
10677            code,
10678            output_tail: String::new(),
10679            duration_ms: 0,
10680            resource_blocked,
10681        }
10682    }
10683
10684    #[test]
10685    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
10686        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
10687        assert!(
10688            !verify_inconclusive(&[outcome(Some(1), false)]),
10689            "an ordinary failure is still evidence about the patch"
10690        );
10691        assert!(verify_inconclusive(&[outcome(None, true)]));
10692        assert!(
10693            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
10694            "one inconclusive outcome taints the whole batch"
10695        );
10696        assert!(!verify_inconclusive(&[]));
10697    }
10698
10699    #[tokio::test]
10700    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
10701        // Alive for the first two checks, then dead - confirms the loop
10702        // actually re-polls rather than deciding once and sleeping out the
10703        // ceiling regardless.
10704        let calls = std::sync::atomic::AtomicUsize::new(0);
10705        let started = Instant::now();
10706        wait_for_pids_with(
10707            &[123],
10708            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
10709            Duration::from_millis(5),
10710            Duration::from_secs(5),
10711        )
10712        .await;
10713        assert!(
10714            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
10715            "must keep checking rather than deciding on the first answer"
10716        );
10717        assert!(
10718            started.elapsed() < Duration::from_secs(1),
10719            "must return the moment it is confirmed dead, not wait out the ceiling"
10720        );
10721    }
10722
10723    #[tokio::test]
10724    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
10725        let started = Instant::now();
10726        wait_for_pids_with(
10727            &[123],
10728            |_| true, // never reports dead
10729            Duration::from_millis(5),
10730            Duration::from_millis(30),
10731        )
10732        .await;
10733        let elapsed = started.elapsed();
10734        assert!(
10735            elapsed >= Duration::from_millis(30),
10736            "must not give up before its own ceiling: {elapsed:?}"
10737        );
10738        assert!(
10739            elapsed < Duration::from_secs(1),
10740            "must not wait past its own ceiling either: {elapsed:?}"
10741        );
10742    }
10743
10744    #[tokio::test]
10745    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
10746        let started = Instant::now();
10747        wait_for_pids_with(
10748            &[],
10749            |_| true,
10750            Duration::from_secs(5),
10751            Duration::from_secs(5),
10752        )
10753        .await;
10754        assert!(
10755            started.elapsed() < Duration::from_millis(200),
10756            "an empty pid list has nothing to confirm"
10757        );
10758    }
10759
10760    // `review_conclusion` is the exact decision the review hand-off task
10761    // fixed: a round budget spent (or a tree that stopped moving) must not
10762    // collapse into `Blocked` regardless of what verification actually
10763    // said. Deterministic and process-free for the same reason the
10764    // `round_is_clean` family above is.
10765    fn review_round(
10766        clean: bool,
10767        blocking: usize,
10768        answered: usize,
10769        expected: usize,
10770        progressed: bool,
10771        e2e_ok: bool,
10772    ) -> ReviewRound {
10773        ReviewRound {
10774            round: 1,
10775            head: "h".to_owned(),
10776            verified_head: None,
10777            verified_at: None,
10778            reviews: Vec::new(),
10779            e2e: vec![CommandOutcome {
10780                command: "test".to_owned(),
10781                code: Some(if e2e_ok { 0 } else { 1 }),
10782                output_tail: String::new(),
10783                duration_ms: 0,
10784                resource_blocked: false,
10785            }],
10786            verify_retried: false,
10787            e2e_deferred: false,
10788            e2e_defer_reason: None,
10789            fix: None,
10790            blocking,
10791            answered,
10792            expected,
10793            clean,
10794            progressed,
10795            vote_split: false,
10796            reconsideration: Vec::new(),
10797            verdict: None,
10798        }
10799    }
10800
10801    #[test]
10802    fn review_conclusion_is_none_when_nothing_has_run() {
10803        assert_eq!(review_conclusion(&[], 3), None);
10804    }
10805
10806    #[test]
10807    fn review_conclusion_is_none_while_rounds_remain() {
10808        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
10809        assert_eq!(review_conclusion(&rounds, 3), None);
10810    }
10811
10812    #[test]
10813    fn review_conclusion_is_gating_once_a_round_is_clean() {
10814        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
10815        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
10816    }
10817
10818    #[test]
10819    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
10820        let rounds = vec![
10821            review_round(false, 1, 2, 2, true, true),
10822            review_round(false, 1, 2, 2, true, true),
10823        ];
10824        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
10825    }
10826
10827    #[test]
10828    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
10829        let rounds = vec![
10830            review_round(false, 1, 2, 2, true, true),
10831            review_round(false, 1, 2, 2, true, false),
10832        ];
10833        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
10834    }
10835
10836    #[test]
10837    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
10838        // Magi never got a command to run against this round's own head — a
10839        // resource-blocked attempt, not a red one — so this must never
10840        // settle on `Blocked` the way a genuine e2e failure would. `None`
10841        // here is what tells `Runner::review_loop` to retry the check
10842        // itself rather than trust this cheap recomputation with a verdict
10843        // it cannot actually produce.
10844        let mut blocked = review_round(false, 1, 2, 2, true, false);
10845        blocked.e2e[0].resource_blocked = true;
10846        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
10847        assert_eq!(review_conclusion(&rounds, 2), None);
10848    }
10849
10850    #[test]
10851    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
10852        // Missing input, not a verified tree — never a hand-off candidate.
10853        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
10854        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
10855    }
10856
10857    #[test]
10858    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
10859        let rounds = vec![
10860            review_round(false, 1, 2, 2, false, true),
10861            review_round(false, 1, 2, 2, false, true),
10862        ];
10863        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
10864    }
10865
10866    fn secs(n: u64) -> Duration {
10867        Duration::from_secs(n)
10868    }
10869
10870    /// A throwaway repo with one commit on `main`, for tests that need `merge`
10871    /// to make real (and, if it runs at all, real*ly fail*) git calls.
10872    fn init_repo(dir: &Path) {
10873        let run = |args: &[&str]| {
10874            let out = std::process::Command::new("git")
10875                .args(args)
10876                .current_dir(dir)
10877                .quiet()
10878                .output()
10879                .expect("spawn git");
10880            assert!(
10881                out.status.success(),
10882                "git {args:?} failed: {}",
10883                String::from_utf8_lossy(&out.stderr)
10884            );
10885        };
10886        run(&["init", "-b", "main"]);
10887        run(&["config", "user.name", "magi test"]);
10888        run(&["config", "user.email", "magi@example.com"]);
10889        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
10890        run(&["add", "-A"]);
10891        run(&["commit", "-m", "init"]);
10892    }
10893
10894    // `settle_questions` is what closes the ghost the phone showed: a run's
10895    // seat asked something, the run then ended, and nothing was left to
10896    // abandon the question it left `open`. `HOME` is a process-wide
10897    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
10898    // first time it runs in the binary — every test below still reaches the
10899    // same directory whichever call won, and each gets its own run id from
10900    // `RunState::new`, so they never collide there.
10901    fn ask_test_home() {
10902        crate::run::pin_test_home();
10903    }
10904
10905    /// A minimal, git-free `Runner` at a given status — `settle_questions`
10906    /// reads nothing else off it.
10907    fn runner_at(status: RunStatus) -> Runner {
10908        let mut state = RunState::new(
10909            PathBuf::from("/nonexistent/repo"),
10910            "main".to_owned(),
10911            "deadbeef".to_owned(),
10912            "task".to_owned(),
10913            Config::default(),
10914        );
10915        state.status = status;
10916        Runner {
10917            state,
10918            roles: ResolvedRoles {
10919                implementers: Vec::new(),
10920                judges: Vec::new(),
10921                reviewers: Vec::new(),
10922                fixer: None,
10923                conductor: conductor(),
10924                implementer_roster: Vec::new(),
10925                judge_roster: Vec::new(),
10926                reviewer_roster: Vec::new(),
10927            },
10928            sem: Arc::new(Semaphore::new(1)),
10929            pause: Pause::new(),
10930            interrupt: Pause::new(),
10931        }
10932    }
10933
10934    /// `park_here` folding in the reason `Pause::park_because` recorded -
10935    /// this is what lets an operator reading a run's events tell an
10936    /// interrupt-driven park from an ordinary shutdown park.
10937    #[test]
10938    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
10939        crate::run::pin_test_home();
10940        let mut runner = runner_at(RunStatus::Implementing);
10941        let interrupt = Pause::new();
10942        runner.watch_interrupt(interrupt.clone());
10943
10944        interrupt.park_because("task a1b2 asked to run first");
10945
10946        assert!(runner.park_here().expect("park_here"));
10947        assert!(runner.state.parked);
10948        let last = runner.state.events.last().expect("a park event");
10949        assert_eq!(last.node, "park");
10950        assert!(
10951            last.message.contains("task a1b2 asked to run first"),
10952            "expected the interrupt reason in {:?}",
10953            last.message
10954        );
10955    }
10956
10957    /// The cause is stored on the pass when it parks, and the first cause a
10958    /// shared pause was given is the one that stays.
10959    #[test]
10960    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
10961        crate::run::pin_test_home();
10962        let mut runner = runner_at(RunStatus::Implementing);
10963        runner.state.begin_pass();
10964        let shutdown = Pause::new();
10965        runner.on_pause(shutdown.clone());
10966        let upgrade = ParkCause::Upgrade {
10967            from: "0.121.0".to_owned(),
10968            to: Some("0.121.1".to_owned()),
10969        };
10970        shutdown.park_for(upgrade.clone());
10971        shutdown.park_for(ParkCause::Operator);
10972
10973        assert!(runner.park_here().expect("park_here"));
10974        assert_eq!(
10975            runner.state.passes[0].end,
10976            Some(crate::run::PassEnd::Parked { cause: upgrade })
10977        );
10978        let last = runner.state.events.last().expect("a park event");
10979        assert!(last.message.contains("0.121.0"), "{}", last.message);
10980    }
10981
10982    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
10983    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
10984    /// cleared) must not make a *different* run - one only watching its own,
10985    /// unshared interrupt `Pause` - see itself as parked. If a future change
10986    /// ever collapsed these back into one handle, the interrupt scheduler
10987    /// would park every run for the rest of the daemon's life, not just the
10988    /// one it meant to interrupt.
10989    #[test]
10990    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
10991        crate::run::pin_test_home();
10992        let mut runner = runner_at(RunStatus::Implementing);
10993        let shutdown = Pause::new();
10994        runner.on_pause(shutdown.clone());
10995        let interrupt = Pause::new();
10996        runner.watch_interrupt(interrupt.clone());
10997
10998        // Nobody has asked for anything yet.
10999        assert!(!runner.park_here().expect("park_here"));
11000        assert!(!runner.state.parked);
11001
11002        // Only the interrupt handle fires; the shutdown handle stays clear.
11003        interrupt.park_because("test");
11004        assert!(!shutdown.parked());
11005        assert!(runner.park_here().expect("park_here"));
11006    }
11007
11008    /// The property every prior attempt at this feature failed to pin down:
11009    /// asking a run to park while one of its nodes has a real, in-flight
11010    /// async operation running (an agent call, in production) must not cut
11011    /// that operation short. `park_here` is only ever consulted *between*
11012    /// `execute`'s node calls - see its own doc - so nothing inside a node
11013    /// can observe a park request until the node itself returns. This proves
11014    /// that structurally, with real `tokio` concurrency and a channel
11015    /// handshake (never a sleep, which would only prove "usually", not
11016    /// "cannot"): the "node" below reports that it has genuinely started,
11017    /// and only then is the park requested; the node still has to be told to
11018    /// finish before `park_here` is ever called, exactly mirroring every
11019    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
11020    /// in `execute`.
11021    #[tokio::test]
11022    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
11023        crate::run::pin_test_home();
11024        let mut runner = runner_at(RunStatus::Implementing);
11025        let interrupt = Pause::new();
11026        runner.watch_interrupt(interrupt.clone());
11027
11028        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
11029        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
11030
11031        // Stands in for one node's in-flight agent call: it proves it has
11032        // genuinely started, then blocks - exactly as a spawned CLI process
11033        // does - until told to finish.
11034        let node = async move {
11035            started_tx.send(()).expect("send started");
11036            finish_rx.await.expect("recv finish");
11037            "node finished"
11038        };
11039
11040        let interrupter = async move {
11041            started_rx.await.expect("recv started");
11042            // The call is now genuinely in flight. Ask it to park.
11043            interrupt.park_because("higher-priority task waiting");
11044            // Nothing the node does can observe this yet - there is no
11045            // check inside it, by construction - so let the executor run
11046            // anything pending and then let the node finish on its own.
11047            tokio::task::yield_now().await;
11048            finish_tx.send(()).expect("send finish");
11049        };
11050
11051        let (node_result, ()) = tokio::join!(node, interrupter);
11052        assert_eq!(
11053            node_result, "node finished",
11054            "the in-flight call ran to completion"
11055        );
11056
11057        // Only now, at the boundary the real `execute` would check right
11058        // after this node, does the park take effect.
11059        assert!(runner.park_here().expect("park_here"));
11060        assert!(runner.state.parked);
11061    }
11062
11063    /// A run parked mid-competition carries every field it had accumulated
11064    /// through the exact same disk round-trip an ordinary resume uses -
11065    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
11066    /// Nothing about parking for an interrupt is a special case of that path;
11067    /// this is what proves it rather than assuming it.
11068    #[test]
11069    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
11070        crate::run::pin_test_home();
11071        let mut runner = runner_at(RunStatus::Judging);
11072        // `Runner::resume` re-resolves roles from the saved config, which
11073        // refuses an empty roster - give it the same minimal one `conductor`
11074        // itself uses.
11075        runner.state.config.agents = vec![conductor()];
11076        runner.state.candidates = vec![Candidate {
11077            index: 0,
11078            label: 'A',
11079            agent: "alpha".to_owned(),
11080            branch: "magi/x/A".to_owned(),
11081            worktree: PathBuf::from("/nonexistent/worktree"),
11082            summary: "did the thing".to_owned(),
11083            stat: "1 file changed".to_owned(),
11084            files: 1,
11085            commits: 1,
11086            empty: false,
11087            failed: None,
11088            verified_noop: None,
11089            duration_ms: 1234,
11090            folded: false,
11091        }];
11092        let run_id = runner.state.id.clone();
11093
11094        let interrupt = Pause::new();
11095        runner.watch_interrupt(interrupt.clone());
11096        interrupt.park_because("task c3d4 asked to run first");
11097        assert!(runner.park_here().expect("park_here"));
11098
11099        let resumed = Runner::resume(&run_id).expect("resume");
11100        assert_eq!(resumed.state.candidates.len(), 1);
11101        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
11102        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
11103        assert_eq!(resumed.state.status, runner.state.status);
11104        assert!(
11105            resumed.state.parked,
11106            "still parked until `execute` actually walks the graph again"
11107        );
11108        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
11109    }
11110
11111    /// A fresh open question on `run`, stored and handed back for assertions.
11112    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
11113        let mut q = ask::Question::new(
11114            run.to_owned(),
11115            "implement".to_owned(),
11116            "impl-A".to_owned(),
11117            "Which storage backend should the cache use?".to_owned(),
11118            String::new(),
11119            vec!["SQLite".to_owned(), "Redis".to_owned()],
11120        );
11121        store.put(&mut q).unwrap();
11122        q
11123    }
11124
11125    #[test]
11126    fn a_failed_runs_open_question_is_abandoned() {
11127        ask_test_home();
11128        let store = ask::Questions::open();
11129        let mut runner = runner_at(RunStatus::Failed);
11130        let run = runner.state.id.clone();
11131        let q = ask_open_question(&store, &run);
11132
11133        runner.settle_questions();
11134
11135        let back = store.get(&q.id).unwrap();
11136        assert!(
11137            !back.status.open(),
11138            "the seat that asked died with the run; nobody is left to read an answer"
11139        );
11140        assert!(
11141            back.detail.contains(&run) && back.detail.contains("failed"),
11142            "the reason names what the run became, not just that it is gone: {}",
11143            back.detail
11144        );
11145    }
11146
11147    #[test]
11148    fn a_merged_runs_open_question_is_abandoned_too() {
11149        ask_test_home();
11150        let store = ask::Questions::open();
11151        // A run that finishes cleanly still leaves nobody to read an answer -
11152        // this is not only a failure-path cleanup.
11153        for status in [RunStatus::Merged, RunStatus::Ready] {
11154            let mut runner = runner_at(status);
11155            let run = runner.state.id.clone();
11156            let q = ask_open_question(&store, &run);
11157
11158            runner.settle_questions();
11159
11160            let back = store.get(&q.id).unwrap();
11161            assert!(
11162                !back.status.open(),
11163                "{status:?} run's question must not outlive the run"
11164            );
11165        }
11166    }
11167
11168    #[test]
11169    fn a_still_resumable_runs_open_question_is_left_alone() {
11170        ask_test_home();
11171        let store = ask::Questions::open();
11172        // `Blocked` and `Stalled` can still be resumed — the candidates, the
11173        // review round and the seat sessions are all still on disk — so a
11174        // question asked mid-round may yet get a real answer from a real
11175        // resume. Sweeping it here would be exactly the failure mode this
11176        // whole feature exists to avoid on the other side.
11177        for status in [RunStatus::Blocked, RunStatus::Stalled] {
11178            let mut runner = runner_at(status);
11179            let run = runner.state.id.clone();
11180            let q = ask_open_question(&store, &run);
11181
11182            runner.settle_questions();
11183
11184            let back = store.get(&q.id).unwrap();
11185            assert!(
11186                back.status.open(),
11187                "{status:?} is still alive; the question must still be waiting"
11188            );
11189        }
11190    }
11191
11192    #[test]
11193    fn settle_questions_never_touches_an_already_answered_question() {
11194        ask_test_home();
11195        let store = ask::Questions::open();
11196        let mut runner = runner_at(RunStatus::Failed);
11197        let run = runner.state.id.clone();
11198        let mut q = ask_open_question(&store, &run);
11199        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
11200            .unwrap();
11201        store.put(&mut q).unwrap();
11202
11203        // Called twice, the way a crash-recovered daemon reclaim and the
11204        // graph's own cleanup both can for the same run — `abandon_for_run`
11205        // only ever touches what is still open, so this must be inert both
11206        // times, not merely the second.
11207        runner.settle_questions();
11208        runner.settle_questions();
11209
11210        let back = store.get(&q.id).unwrap();
11211        assert_eq!(
11212            back.status,
11213            ask::QuestionStatus::Answered,
11214            "a real answer is a decision on record, never overwritten by a sweep"
11215        );
11216    }
11217
11218    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
11219    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
11220    /// without merging, whose winner is still the operator's answer to read.
11221    /// Nothing previously called `fold_run` itself with a real `tally`, so
11222    /// this is the first test to pin down the one distinction the whole
11223    /// automatic-fold feature depends on: the winner's worktree and branch
11224    /// must survive, everything else sharing the run's worktree bay - a
11225    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
11226    /// second sweep treats every non-winner directory under the bay alike -
11227    /// must not.
11228    #[tokio::test]
11229    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
11230        crate::run::pin_test_home();
11231        let tmp = tempfile::tempdir().expect("tempdir");
11232        let repo = tmp.path().join("repo");
11233        std::fs::create_dir_all(&repo).unwrap();
11234        init_repo(&repo);
11235
11236        let mut config = Config::default();
11237        config.graph.worktree_root = Some(tmp.path().join("wt"));
11238
11239        let mut state = RunState::new(
11240            repo.clone(),
11241            "main".to_owned(),
11242            "deadbeef".to_owned(),
11243            "task".to_owned(),
11244            config,
11245        );
11246        let root = state.worktree_root();
11247        let wt_a = root.join("cand-A");
11248        let wt_b = root.join("cand-B");
11249        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
11250            .await
11251            .expect("worktree A");
11252        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
11253            .await
11254            .expect("worktree B");
11255
11256        state.candidates = vec![
11257            Candidate {
11258                index: 0,
11259                label: 'A',
11260                agent: "alpha".to_owned(),
11261                branch: "magi/x/A".to_owned(),
11262                worktree: wt_a.clone(),
11263                summary: String::new(),
11264                stat: String::new(),
11265                files: 0,
11266                commits: 0,
11267                empty: false,
11268                failed: None,
11269                verified_noop: None,
11270                duration_ms: 0,
11271                folded: false,
11272            },
11273            Candidate {
11274                index: 1,
11275                label: 'B',
11276                agent: "beta".to_owned(),
11277                branch: "magi/x/B".to_owned(),
11278                worktree: wt_b.clone(),
11279                summary: String::new(),
11280                stat: String::new(),
11281                files: 0,
11282                commits: 0,
11283                empty: false,
11284                failed: None,
11285                verified_noop: None,
11286                duration_ms: 0,
11287                folded: false,
11288            },
11289        ];
11290        state.tally = Some(Tally {
11291            first_choice: BTreeMap::from([('A', 1)]),
11292            borda: BTreeMap::new(),
11293            winner: 'A',
11294            rankings: 1,
11295            unanimous_initial: true,
11296            deliberated: false,
11297            changed_votes: 0,
11298            unanimous_final: true,
11299            tie_break: None,
11300            judges: 1,
11301            present: 1,
11302            quorum: 1,
11303            met_quorum: true,
11304            uncontested: None,
11305        });
11306        state.status = RunStatus::Ready;
11307
11308        fold_run(&mut state, false, &crate::run::home())
11309            .await
11310            .expect("fold_run");
11311
11312        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
11313        assert!(
11314            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
11315            "the unmerged winner's branch survives"
11316        );
11317        assert!(
11318            !state.candidates[0].folded,
11319            "the winner is not marked folded"
11320        );
11321
11322        assert!(!wt_b.exists(), "the loser's worktree is removed");
11323        assert!(
11324            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
11325            "the loser's branch is removed"
11326        );
11327        assert!(state.candidates[1].folded, "the loser is marked folded");
11328    }
11329
11330    /// A branch handed to a later run is that run's (and its pull request's):
11331    /// folding the run that released it must not delete it.
11332    #[tokio::test]
11333    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
11334        let tmp = tempfile::tempdir().expect("tempdir");
11335        let repo = tmp.path().join("repo");
11336        std::fs::create_dir_all(&repo).unwrap();
11337        init_repo(&repo);
11338        let home = tmp.path().join("home");
11339
11340        let mut config = Config::default();
11341        config.graph.worktree_root = Some(tmp.path().join("wt"));
11342        let mut state = RunState::new(
11343            repo.clone(),
11344            "main".to_owned(),
11345            "deadbeef".to_owned(),
11346            "task".to_owned(),
11347            config,
11348        );
11349        // The worktree is already gone (released); the branch survives.
11350        git::git(&repo, &["branch", "magi/x/A", "main"])
11351            .await
11352            .expect("branch");
11353        state.candidates = vec![Candidate {
11354            index: 0,
11355            label: 'A',
11356            agent: "alpha".to_owned(),
11357            branch: "magi/x/A".to_owned(),
11358            worktree: state.worktree_root().join("cand-A"),
11359            summary: String::new(),
11360            stat: String::new(),
11361            files: 0,
11362            commits: 0,
11363            empty: false,
11364            failed: None,
11365            verified_noop: None,
11366            duration_ms: 0,
11367            folded: true,
11368        }];
11369        state.released_to = Some("20260901-000000-new1".to_owned());
11370        state.released_branches = vec!["magi/x/A".to_owned()];
11371
11372        fold_run(&mut state, true, &home).await.expect("fold_run");
11373
11374        assert!(
11375            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
11376            "the handed-over branch survives a fold"
11377        );
11378    }
11379
11380    /// A winner with nothing ahead of the base is caught before `gh` is ever
11381    /// asked for a pull request, and the message carries what the task's
11382    /// references resolved to.
11383    #[tokio::test]
11384    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
11385        let tmp = tempfile::tempdir().expect("tempdir");
11386        let repo = tmp.path().join("repo");
11387        std::fs::create_dir_all(&repo).unwrap();
11388        init_repo(&repo);
11389        let run = |args: &[&str]| {
11390            let out = std::process::Command::new("git")
11391                .quiet()
11392                .args(args)
11393                .current_dir(&repo)
11394                .output()
11395                .expect("spawn git");
11396            assert!(out.status.success(), "git {args:?}");
11397        };
11398        run(&["branch", "magi/x/A"]);
11399        run(&["checkout", "-q", "-b", "magi/x/B"]);
11400        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
11401        run(&["add", "-A"]);
11402        run(&["commit", "-q", "-m", "work"]);
11403        run(&["checkout", "-q", "main"]);
11404
11405        // A pull request is compared against the remote's base, so the
11406        // fixture needs one. Before it exists nothing can be read, and the
11407        // answer must be "not empty".
11408        let probe = RunState::new(
11409            repo.clone(),
11410            "main".to_owned(),
11411            "deadbeef".to_owned(),
11412            "task".to_owned(),
11413            Config::default(),
11414        );
11415        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
11416        let bare = tmp.path().join("origin.git");
11417        let out = std::process::Command::new("git")
11418            .quiet()
11419            .args(["init", "-q", "--bare"])
11420            .arg(&bare)
11421            .output()
11422            .expect("spawn git");
11423        assert!(out.status.success(), "git init --bare");
11424        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
11425        run(&["push", "-q", "origin", "main"]);
11426
11427        let mut state = RunState::new(
11428            repo.clone(),
11429            "main".to_owned(),
11430            "deadbeef".to_owned(),
11431            "task".to_owned(),
11432            Config::default(),
11433        );
11434        state.seeds = vec![refs::Seed {
11435            token: "magi/27b2/A".to_owned(),
11436            kind: refs::SeedKind::Unresolved,
11437            sha: String::new(),
11438            branch: true,
11439            detail: "no branch or commit named magi/27b2/A".to_owned(),
11440        }];
11441
11442        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
11443        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
11444        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
11445        let detail = empty_candidate_detail(&state, "main");
11446        assert!(detail.starts_with("empty candidate"), "{detail}");
11447        assert!(detail.contains("magi/27b2/A"), "{detail}");
11448    }
11449
11450    /// `status == Ready` used to be read as "this is the harmless
11451    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
11452    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
11453    /// run's PR was closed without merging — and reentering `merge` with
11454    /// `mode` still `Pr` does not know the difference, so it pushed and
11455    /// opened a second pull request. `mode == Local` reproduces the same
11456    /// blind spot without a network call: reentry must not attempt another
11457    /// git merge once this node has already recorded an outcome.
11458    #[tokio::test]
11459    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
11460        let tmp = tempfile::tempdir().expect("tempdir");
11461        let repo = tmp.path().join("repo");
11462        std::fs::create_dir_all(&repo).unwrap();
11463        init_repo(&repo);
11464
11465        let mut config = Config::default();
11466        config.merge.mode = MergeMode::Local;
11467
11468        let mut state = RunState::new(
11469            repo.clone(),
11470            "main".to_owned(),
11471            "deadbeef".to_owned(),
11472            "task".to_owned(),
11473            config,
11474        );
11475        state.candidates = vec![Candidate {
11476            index: 0,
11477            label: 'A',
11478            agent: "alpha".to_owned(),
11479            branch: "does-not-exist".to_owned(),
11480            worktree: repo.clone(),
11481            summary: String::new(),
11482            stat: String::new(),
11483            files: 0,
11484            commits: 0,
11485            empty: false,
11486            failed: None,
11487            verified_noop: None,
11488            duration_ms: 0,
11489            folded: false,
11490        }];
11491        state.tally = Some(Tally {
11492            first_choice: BTreeMap::from([('A', 1)]),
11493            borda: BTreeMap::new(),
11494            winner: 'A',
11495            rankings: 1,
11496            unanimous_initial: true,
11497            deliberated: false,
11498            changed_votes: 0,
11499            unanimous_final: true,
11500            tie_break: None,
11501            judges: 0,
11502            present: 0,
11503            quorum: 0,
11504            met_quorum: true,
11505            uncontested: Some("only candidate A produced a change".to_owned()),
11506        });
11507        state.reviews = vec![ReviewRound {
11508            round: 1,
11509            head: "deadbeef".to_owned(),
11510            verified_head: None,
11511            verified_at: None,
11512            reviews: Vec::new(),
11513            e2e: Vec::new(),
11514            fix: None,
11515            blocking: 0,
11516            answered: 0,
11517            expected: 0,
11518            clean: true,
11519            verify_retried: false,
11520            e2e_deferred: false,
11521            e2e_defer_reason: None,
11522            progressed: false,
11523            vote_split: false,
11524            reconsideration: Vec::new(),
11525            verdict: None,
11526        }];
11527        state.gate = vec![CommandOutcome {
11528            command: "test".to_owned(),
11529            code: Some(0),
11530            output_tail: String::new(),
11531            duration_ms: 0,
11532            resource_blocked: false,
11533        }];
11534        state.gate_ran = true;
11535        // Reached its conclusion already — e.g. `land` closing the PR without
11536        // merging it, which (like the honest `MergeMode::None` path) leaves
11537        // `status` at `Ready`. The recorded outcome is what actually marks
11538        // this node done.
11539        state.status = RunStatus::Ready;
11540        state.merge = Some(MergeOutcome {
11541            mode: MergeMode::Local,
11542            ok: false,
11543            detail: "already concluded".to_owned(),
11544            empty: false,
11545        });
11546
11547        let mut runner = Runner {
11548            state,
11549            roles: ResolvedRoles {
11550                implementers: Vec::new(),
11551                judges: Vec::new(),
11552                reviewers: Vec::new(),
11553                fixer: None,
11554                conductor: conductor(),
11555                implementer_roster: Vec::new(),
11556                judge_roster: Vec::new(),
11557                reviewer_roster: Vec::new(),
11558            },
11559            sem: Arc::new(Semaphore::new(1)),
11560            pause: Pause::new(),
11561            interrupt: Pause::new(),
11562        };
11563
11564        runner.merge().await.expect("merge");
11565
11566        assert_eq!(
11567            runner.state.status,
11568            RunStatus::Ready,
11569            "a concluded run's status must not change on reentry"
11570        );
11571        assert_eq!(
11572            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
11573            Some("already concluded"),
11574            "merge must not run again once the node already recorded an outcome"
11575        );
11576    }
11577
11578    /// `gate` leaves `state.gate_ran` false both before it has ever run and
11579    /// when its last attempt was resource-blocked (the shared build cache
11580    /// could not be acquired or confirmed fresh in time - see
11581    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
11582    /// `Vec` this also leaves behind used to read as "nothing failed" and let
11583    /// a run merge a tree the gate never actually checked - exactly the case
11584    /// a contended cache produces on every retry until it clears. `merge`
11585    /// must refuse until `gate` has actually recorded an attempt.
11586    #[tokio::test]
11587    async fn merge_refuses_a_gate_that_has_not_actually_run() {
11588        let tmp = tempfile::tempdir().expect("tempdir");
11589        let repo = tmp.path().join("repo");
11590        std::fs::create_dir_all(&repo).unwrap();
11591        init_repo(&repo);
11592
11593        let mut config = Config::default();
11594        config.merge.mode = MergeMode::Local;
11595
11596        let mut state = RunState::new(
11597            repo.clone(),
11598            "main".to_owned(),
11599            "deadbeef".to_owned(),
11600            "task".to_owned(),
11601            config,
11602        );
11603        state.candidates = vec![Candidate {
11604            index: 0,
11605            label: 'A',
11606            agent: "alpha".to_owned(),
11607            branch: "does-not-exist".to_owned(),
11608            worktree: repo.clone(),
11609            summary: String::new(),
11610            stat: String::new(),
11611            files: 0,
11612            commits: 0,
11613            empty: false,
11614            failed: None,
11615            verified_noop: None,
11616            duration_ms: 0,
11617            folded: false,
11618        }];
11619        state.tally = Some(Tally {
11620            first_choice: BTreeMap::from([('A', 1)]),
11621            borda: BTreeMap::new(),
11622            winner: 'A',
11623            rankings: 1,
11624            unanimous_initial: true,
11625            deliberated: false,
11626            changed_votes: 0,
11627            unanimous_final: true,
11628            tie_break: None,
11629            judges: 0,
11630            present: 0,
11631            quorum: 0,
11632            met_quorum: true,
11633            uncontested: Some("only candidate A produced a change".to_owned()),
11634        });
11635        state.reviews = vec![ReviewRound {
11636            round: 1,
11637            head: "deadbeef".to_owned(),
11638            verified_head: None,
11639            verified_at: None,
11640            reviews: Vec::new(),
11641            e2e: Vec::new(),
11642            fix: None,
11643            blocking: 0,
11644            answered: 0,
11645            expected: 0,
11646            clean: true,
11647            verify_retried: false,
11648            e2e_deferred: false,
11649            e2e_defer_reason: None,
11650            progressed: false,
11651            vote_split: false,
11652            reconsideration: Vec::new(),
11653            verdict: None,
11654        }];
11655        // The point: `gate` has not recorded anything yet.
11656        state.gate = Vec::new();
11657        state.gate_ran = false;
11658        state.status = RunStatus::Gating;
11659
11660        let mut runner = Runner {
11661            state,
11662            roles: ResolvedRoles {
11663                implementers: Vec::new(),
11664                judges: Vec::new(),
11665                reviewers: Vec::new(),
11666                fixer: None,
11667                conductor: conductor(),
11668                implementer_roster: Vec::new(),
11669                judge_roster: Vec::new(),
11670                reviewer_roster: Vec::new(),
11671            },
11672            sem: Arc::new(Semaphore::new(1)),
11673            pause: Pause::new(),
11674            interrupt: Pause::new(),
11675        };
11676
11677        runner.merge().await.expect("merge");
11678
11679        assert!(
11680            runner.state.merge.is_none(),
11681            "an empty gate must never be read as a passing one: {:?}",
11682            runner.state.merge
11683        );
11684    }
11685
11686    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
11687    /// commands configured and `merge.mode` is `none` (a review-only run).
11688    /// `gate` must still record a real attempt — zero commands, vacuously
11689    /// passed — rather than leaving `state.gate` empty in a way `merge`
11690    /// cannot tell apart from "never ran"; otherwise the run reaches
11691    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
11692    #[tokio::test]
11693    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
11694        let tmp = tempfile::tempdir().expect("tempdir");
11695        let repo = tmp.path().join("repo");
11696        std::fs::create_dir_all(&repo).unwrap();
11697        init_repo(&repo);
11698
11699        // Default config: `verify.gate` empty, `merge.mode` is `none`.
11700        let config = Config::default();
11701
11702        let mut state = RunState::new(
11703            repo.clone(),
11704            "main".to_owned(),
11705            "deadbeef".to_owned(),
11706            "task".to_owned(),
11707            config,
11708        );
11709        state.candidates = vec![Candidate {
11710            index: 0,
11711            label: 'A',
11712            agent: "alpha".to_owned(),
11713            branch: "does-not-exist".to_owned(),
11714            worktree: repo.clone(),
11715            summary: String::new(),
11716            stat: String::new(),
11717            files: 0,
11718            commits: 0,
11719            empty: false,
11720            failed: None,
11721            verified_noop: None,
11722            duration_ms: 0,
11723            folded: false,
11724        }];
11725        state.tally = Some(Tally {
11726            first_choice: BTreeMap::from([('A', 1)]),
11727            borda: BTreeMap::new(),
11728            winner: 'A',
11729            rankings: 1,
11730            unanimous_initial: true,
11731            deliberated: false,
11732            changed_votes: 0,
11733            unanimous_final: true,
11734            tie_break: None,
11735            judges: 0,
11736            present: 0,
11737            quorum: 0,
11738            met_quorum: true,
11739            uncontested: Some("only candidate A produced a change".to_owned()),
11740        });
11741        state.reviews = vec![ReviewRound {
11742            round: 1,
11743            head: "deadbeef".to_owned(),
11744            verified_head: None,
11745            verified_at: None,
11746            reviews: Vec::new(),
11747            e2e: Vec::new(),
11748            fix: None,
11749            blocking: 0,
11750            answered: 0,
11751            expected: 0,
11752            clean: true,
11753            verify_retried: false,
11754            e2e_deferred: false,
11755            e2e_defer_reason: None,
11756            progressed: false,
11757            vote_split: false,
11758            reconsideration: Vec::new(),
11759            verdict: None,
11760        }];
11761
11762        let mut runner = Runner {
11763            state,
11764            roles: ResolvedRoles {
11765                implementers: Vec::new(),
11766                judges: Vec::new(),
11767                reviewers: Vec::new(),
11768                fixer: None,
11769                conductor: conductor(),
11770                implementer_roster: Vec::new(),
11771                judge_roster: Vec::new(),
11772                reviewer_roster: Vec::new(),
11773            },
11774            sem: Arc::new(Semaphore::new(1)),
11775            pause: Pause::new(),
11776            interrupt: Pause::new(),
11777        };
11778
11779        runner.gate().await.expect("gate");
11780        assert!(
11781            runner.state.gate_ran,
11782            "zero configured commands is still a real attempt, not an unrun gate"
11783        );
11784        assert!(runner.state.gate.is_empty());
11785        assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
11786        assert_ne!(
11787            runner.state.status,
11788            RunStatus::Blocked,
11789            "a gate with nothing to check must not read as failed"
11790        );
11791
11792        runner.merge().await.expect("merge");
11793        assert_eq!(
11794            runner.state.status,
11795            RunStatus::Ready,
11796            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
11797        );
11798    }
11799
11800    /// `Config::cache_dir` is derived from `verify.e2e` as well as
11801    /// `verify.gate` (so the e2e leg and the final gate never build against
11802    /// different directories). With zero `verify.gate` commands but a
11803    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
11804    /// that lease before discovering it had nothing to run - so a repo with
11805    /// no gate commands could come back `resource_blocked` (and therefore
11806    /// still `gate_ran == false`) on nothing but an unrelated run holding the
11807    /// cache, exactly the contention this run's own zero commands could
11808    /// never have touched. `gate` must recognise there is nothing to check
11809    /// before it ever asks for the lease.
11810    #[tokio::test]
11811    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
11812        crate::run::pin_test_home();
11813        let home = crate::run::home();
11814
11815        let tmp = tempfile::tempdir().expect("tempdir");
11816        let repo = tmp.path().join("repo");
11817        std::fs::create_dir_all(&repo).unwrap();
11818        init_repo(&repo);
11819        // Unique to this test, so holding its lease cannot collide with
11820        // another test sharing the same process-wide `home`.
11821        let cache_dir = tmp.path().join("target");
11822
11823        let mut config = Config::default();
11824        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
11825        // `verify.gate` stays empty (the default). Bounded so a regression
11826        // that does start waiting fails the test in seconds, not hangs it.
11827        config.graph.timeout_verify = Some(2);
11828
11829        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
11830        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
11831            .expect("no io error acquiring directly")
11832        {
11833            crate::cache::AcquireOutcome::Acquired(g) => g,
11834            crate::cache::AcquireOutcome::Busy(b) => {
11835                panic!("expected the direct acquire to win the lease first: {b:?}")
11836            }
11837        };
11838
11839        let mut state = RunState::new(
11840            repo.clone(),
11841            "main".to_owned(),
11842            "deadbeef".to_owned(),
11843            "task".to_owned(),
11844            config,
11845        );
11846        state.candidates = vec![Candidate {
11847            index: 0,
11848            label: 'A',
11849            agent: "alpha".to_owned(),
11850            branch: "does-not-exist".to_owned(),
11851            worktree: repo.clone(),
11852            summary: String::new(),
11853            stat: String::new(),
11854            files: 0,
11855            commits: 0,
11856            empty: false,
11857            failed: None,
11858            verified_noop: None,
11859            duration_ms: 0,
11860            folded: false,
11861        }];
11862        state.tally = Some(Tally {
11863            first_choice: BTreeMap::from([('A', 1)]),
11864            borda: BTreeMap::new(),
11865            winner: 'A',
11866            rankings: 1,
11867            unanimous_initial: true,
11868            deliberated: false,
11869            changed_votes: 0,
11870            unanimous_final: true,
11871            tie_break: None,
11872            judges: 0,
11873            present: 0,
11874            quorum: 0,
11875            met_quorum: true,
11876            uncontested: Some("only candidate A produced a change".to_owned()),
11877        });
11878        state.reviews = vec![ReviewRound {
11879            round: 1,
11880            head: "deadbeef".to_owned(),
11881            verified_head: None,
11882            verified_at: None,
11883            reviews: Vec::new(),
11884            e2e: Vec::new(),
11885            fix: None,
11886            blocking: 0,
11887            answered: 0,
11888            expected: 0,
11889            clean: true,
11890            verify_retried: false,
11891            e2e_deferred: false,
11892            e2e_defer_reason: None,
11893            progressed: false,
11894            vote_split: false,
11895            reconsideration: Vec::new(),
11896            verdict: None,
11897        }];
11898
11899        let mut runner = Runner {
11900            state,
11901            roles: ResolvedRoles {
11902                implementers: Vec::new(),
11903                judges: Vec::new(),
11904                reviewers: Vec::new(),
11905                fixer: None,
11906                conductor: conductor(),
11907                implementer_roster: Vec::new(),
11908                judge_roster: Vec::new(),
11909                reviewer_roster: Vec::new(),
11910            },
11911            sem: Arc::new(Semaphore::new(1)),
11912            pause: Pause::new(),
11913            interrupt: Pause::new(),
11914        };
11915
11916        let started = std::time::Instant::now();
11917        runner.gate().await.expect("gate");
11918        assert!(
11919            started.elapsed() < Duration::from_secs(1),
11920            "a gate with nothing to run must never wait on a lease it never needed"
11921        );
11922        assert!(
11923            runner.state.gate_ran,
11924            "zero commands is still a real, immediate attempt"
11925        );
11926        assert!(runner.state.gate.is_empty());
11927        assert_ne!(
11928            runner.state.status,
11929            RunStatus::Blocked,
11930            "must not read as resource-blocked on a lease it never asked for"
11931        );
11932    }
11933
11934    /// The addendum's second gap: a `verify.gate` command running for real
11935    /// wall-clock time had nothing at all to show for it in `active` before
11936    /// `run_commands` learned to record it — a run could sit in `Gating` for
11937    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
11938    /// what was actually happening. Proven with a genuinely still-running
11939    /// command, not just a before/after check on the final state: a poller
11940    /// task reads the same `run.json` `gate()` is writing, the same way the
11941    /// phone or `magi show` would, while the shell command is still blocked
11942    /// on its own release marker.
11943    #[tokio::test]
11944    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
11945        crate::run::pin_test_home();
11946
11947        let tmp = tempfile::tempdir().expect("tempdir");
11948        let repo = tmp.path().join("repo");
11949        std::fs::create_dir_all(&repo).unwrap();
11950        init_repo(&repo);
11951
11952        let mut config = Config::default();
11953        config.verify.gate = vec![
11954            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
11955             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
11956                .to_owned(),
11957        ];
11958
11959        let mut state = RunState::new(
11960            repo.clone(),
11961            "main".to_owned(),
11962            "deadbeef".to_owned(),
11963            "task".to_owned(),
11964            config,
11965        );
11966        let run_id = state.id.clone();
11967        state.candidates = vec![Candidate {
11968            index: 0,
11969            label: 'A',
11970            agent: "alpha".to_owned(),
11971            branch: "does-not-exist".to_owned(),
11972            worktree: repo.clone(),
11973            summary: String::new(),
11974            stat: String::new(),
11975            files: 0,
11976            commits: 0,
11977            empty: false,
11978            failed: None,
11979            verified_noop: None,
11980            duration_ms: 0,
11981            folded: false,
11982        }];
11983        state.tally = Some(Tally {
11984            first_choice: BTreeMap::from([('A', 1)]),
11985            borda: BTreeMap::new(),
11986            winner: 'A',
11987            rankings: 1,
11988            unanimous_initial: true,
11989            deliberated: false,
11990            changed_votes: 0,
11991            unanimous_final: true,
11992            tie_break: None,
11993            judges: 0,
11994            present: 0,
11995            quorum: 0,
11996            met_quorum: true,
11997            uncontested: Some("only candidate A produced a change".to_owned()),
11998        });
11999        state.reviews = vec![ReviewRound {
12000            round: 1,
12001            head: "deadbeef".to_owned(),
12002            verified_head: None,
12003            verified_at: None,
12004            reviews: Vec::new(),
12005            e2e: Vec::new(),
12006            fix: None,
12007            blocking: 0,
12008            answered: 0,
12009            expected: 0,
12010            clean: true,
12011            verify_retried: false,
12012            e2e_deferred: false,
12013            e2e_defer_reason: None,
12014            progressed: false,
12015            vote_split: false,
12016            reconsideration: Vec::new(),
12017            verdict: None,
12018        }];
12019
12020        let mut runner = Runner {
12021            state,
12022            roles: ResolvedRoles {
12023                implementers: Vec::new(),
12024                judges: Vec::new(),
12025                reviewers: Vec::new(),
12026                fixer: None,
12027                conductor: conductor(),
12028                implementer_roster: Vec::new(),
12029                judge_roster: Vec::new(),
12030                reviewer_roster: Vec::new(),
12031            },
12032            sem: Arc::new(Semaphore::new(1)),
12033            pause: Pause::new(),
12034            interrupt: Pause::new(),
12035        };
12036
12037        let started_marker = repo.join("started.marker");
12038        let release_marker = repo.join("release.marker");
12039        let poller = tokio::spawn(async move {
12040            // Bounded so a regression that never records the task entry
12041            // fails this test in seconds instead of hanging the suite —
12042            // the same shape `a_park_requested_while_a_seat_is_mid_call_
12043            // does_not_cut_it_short` uses for the same reason.
12044            for _ in 0..100 {
12045                if started_marker.exists()
12046                    && let Ok(s) = crate::run::RunState::load(&run_id)
12047                    && let Some(a) = s.active.get("gate")
12048                {
12049                    std::fs::write(&release_marker, b"go").expect("release marker");
12050                    return Some(a.clone());
12051                }
12052                tokio::time::sleep(Duration::from_millis(50)).await;
12053            }
12054            None
12055        });
12056
12057        runner.gate().await.expect("gate");
12058        let captured = poller.await.expect("poller task");
12059        let captured = captured.expect(
12060            "the poller never saw a `gate` task entry in run.json while the command was \
12061             still blocked on its own release marker",
12062        );
12063
12064        assert_eq!(captured.task.as_deref(), Some("gate"));
12065        assert_eq!(captured.node, "gate");
12066        assert_eq!(captured.index, Some(1));
12067        assert_eq!(captured.total, Some(1));
12068        assert!(
12069            captured
12070                .command
12071                .as_deref()
12072                .is_some_and(|c| c.contains("started.marker")),
12073            "{captured:?}"
12074        );
12075
12076        assert!(
12077            runner.state.active.is_empty(),
12078            "the entry must be cleared once the command actually finished: {:?}",
12079            runner.state.active
12080        );
12081        assert!(runner.state.gate_ran);
12082        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
12083    }
12084
12085    /// The hand-off over a blocking finding a reviewer rejected on leaves a
12086    /// record for `land`; one with only a Minor, or no reject, leaves none.
12087    #[tokio::test]
12088    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
12089        use crate::verdict::{Finding, ReviewVote, Severity};
12090        crate::run::pin_test_home();
12091        let tmp = tempfile::tempdir().expect("tempdir");
12092        let repo = tmp.path().join("repo");
12093        std::fs::create_dir_all(&repo).unwrap();
12094        init_repo(&repo);
12095
12096        for (severity, vote, expect) in [
12097            (Severity::Major, ReviewVote::Reject, true),
12098            (Severity::Minor, ReviewVote::Reject, false),
12099            (Severity::Major, ReviewVote::Approve, false),
12100        ] {
12101            let mut round = review_round(false, 1, 1, 1, false, true);
12102            round.reviews = vec![ReviewRecord {
12103                reviewer: 1,
12104                agent: "alpha".to_owned(),
12105                summary: String::new(),
12106                findings: vec![Finding {
12107                    id: "R1-1-1".to_owned(),
12108                    severity,
12109                    file: None,
12110                    line: None,
12111                    title: "t".to_owned(),
12112                    detail: String::new(),
12113                }],
12114                vote: Some(vote),
12115                failed: None,
12116                duration_ms: 0,
12117                attempts: 0,
12118            }];
12119            let mut state = RunState::new(
12120                repo.clone(),
12121                "main".to_owned(),
12122                "deadbeef".to_owned(),
12123                "task".to_owned(),
12124                Config::default(),
12125            );
12126            state.reviews = vec![round];
12127            let mut runner = Runner {
12128                state,
12129                roles: ResolvedRoles {
12130                    implementers: Vec::new(),
12131                    judges: Vec::new(),
12132                    reviewers: Vec::new(),
12133                    fixer: None,
12134                    conductor: conductor(),
12135                    implementer_roster: Vec::new(),
12136                    judge_roster: Vec::new(),
12137                    reviewer_roster: Vec::new(),
12138                },
12139                sem: Arc::new(Semaphore::new(1)),
12140                pause: Pause::new(),
12141                interrupt: Pause::new(),
12142            };
12143            let shell = runner.state.config.shell();
12144            runner
12145                .stop_reviewing("round budget spent", &shell, &repo)
12146                .await
12147                .expect("stop_reviewing");
12148            assert_eq!(runner.state.status, RunStatus::Gating);
12149            assert_eq!(
12150                runner.state.contested_handoff.is_some(),
12151                expect,
12152                "{severity:?} + {vote:?}"
12153            );
12154        }
12155    }
12156
12157    /// The shape the incident this whole fix responds to actually had: the
12158    /// round budget spent, the last round's own e2e blocked on the shared
12159    /// build cache (held here by a live pid — this test process — exactly
12160    /// `cache`'s own unit tests' pattern for "another owner, still alive"
12161    /// without forking a process). `stop_reviewing` must retry it — not
12162    /// silently leave the round looking untouched (the catch-up-only half of
12163    /// the bug), and not read the contention as a red `e2e` and block the
12164    /// run on it (the other half). Called directly, the same way
12165    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
12166    /// above exercises `gate`, so this never needs a real cargo build to
12167    /// reach: the lease is never released, so `with_cache_lease` never gets
12168    /// past acquiring it into anything that would need a real workspace.
12169    #[tokio::test]
12170    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
12171        crate::run::pin_test_home();
12172        let home = crate::run::home();
12173
12174        let tmp = tempfile::tempdir().expect("tempdir");
12175        let repo = tmp.path().join("repo");
12176        std::fs::create_dir_all(&repo).unwrap();
12177        init_repo(&repo);
12178        let head = crate::git::rev_parse(&repo, "HEAD")
12179            .await
12180            .expect("rev-parse");
12181        // Unique to this test, so holding its lease cannot collide with
12182        // another test sharing the same process-wide `home`.
12183        let cache_dir = tmp.path().join("target");
12184
12185        let mut config = Config::default();
12186        config.verify.e2e = vec![format!(
12187            "CARGO_TARGET_DIR='{}' test -f README.md",
12188            cache_dir.display()
12189        )];
12190        config.graph.review_rounds = 1;
12191        // Bounded so a regression that does start waiting fails the test in
12192        // seconds, not hangs it.
12193        config.graph.timeout_verify = Some(2);
12194
12195        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
12196        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
12197            .expect("no io error acquiring directly")
12198        {
12199            crate::cache::AcquireOutcome::Acquired(g) => g,
12200            crate::cache::AcquireOutcome::Busy(b) => {
12201                panic!("expected the direct acquire to win the lease first: {b:?}")
12202            }
12203        };
12204
12205        let mut state = RunState::new(
12206            repo.clone(),
12207            "main".to_owned(),
12208            head.clone(),
12209            "task".to_owned(),
12210            config,
12211        );
12212        state.candidates = vec![Candidate {
12213            index: 0,
12214            label: 'A',
12215            agent: "alpha".to_owned(),
12216            branch: "does-not-exist".to_owned(),
12217            worktree: repo.clone(),
12218            summary: String::new(),
12219            stat: String::new(),
12220            files: 0,
12221            commits: 0,
12222            empty: false,
12223            failed: None,
12224            verified_noop: None,
12225            duration_ms: 0,
12226            folded: false,
12227        }];
12228        state.tally = Some(Tally {
12229            first_choice: BTreeMap::from([('A', 1)]),
12230            borda: BTreeMap::new(),
12231            winner: 'A',
12232            rankings: 1,
12233            unanimous_initial: true,
12234            deliberated: false,
12235            changed_votes: 0,
12236            unanimous_final: true,
12237            tie_break: None,
12238            judges: 0,
12239            present: 0,
12240            quorum: 0,
12241            met_quorum: true,
12242            uncontested: Some("only candidate A produced a change".to_owned()),
12243        });
12244        // The round budget's last round, deferred: `needs_catchup_run`'s
12245        // other trigger. `stop_reviewing`'s retry machinery must treat this
12246        // exactly like a resource-blocked attempt once it actually runs.
12247        state.reviews = vec![ReviewRound {
12248            round: 1,
12249            head: head.clone(),
12250            verified_head: None,
12251            verified_at: None,
12252            reviews: Vec::new(),
12253            e2e: Vec::new(),
12254            fix: None,
12255            blocking: 1,
12256            answered: 1,
12257            expected: 1,
12258            clean: false,
12259            verify_retried: false,
12260            e2e_deferred: true,
12261            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
12262            progressed: false,
12263            vote_split: false,
12264            reconsideration: Vec::new(),
12265            verdict: None,
12266        }];
12267
12268        let mut runner = Runner {
12269            state,
12270            roles: ResolvedRoles {
12271                implementers: Vec::new(),
12272                judges: Vec::new(),
12273                reviewers: Vec::new(),
12274                fixer: None,
12275                conductor: conductor(),
12276                implementer_roster: Vec::new(),
12277                judge_roster: Vec::new(),
12278                reviewer_roster: Vec::new(),
12279            },
12280            sem: Arc::new(Semaphore::new(1)),
12281            pause: Pause::new(),
12282            interrupt: Pause::new(),
12283        };
12284
12285        let shell = runner.state.config.shell();
12286        runner
12287            .stop_reviewing("round budget spent", &shell, &repo)
12288            .await
12289            .expect("stop_reviewing");
12290
12291        let last = runner.state.reviews.last().expect("round record");
12292        assert_eq!(
12293            last.e2e_status(),
12294            E2eStatus::ResourceBlocked,
12295            "the shared cache is still held; the attempt must read as blocked, not deferred or \
12296             failed: {last:?}"
12297        );
12298        assert_eq!(
12299            last.verified_head.as_deref(),
12300            Some(head.as_str()),
12301            "which commit this attempt targeted is known even though nothing finished checking \
12302             it"
12303        );
12304        let first_attempt_at = last
12305            .verified_at
12306            .expect("when this attempt ran is known too");
12307        assert_ne!(
12308            runner.state.status,
12309            RunStatus::Blocked,
12310            "contention is evidence about the machine, not the patch — it must not settle the \
12311             run as blocked: {:?}",
12312            runner.state.status
12313        );
12314        assert!(
12315            !runner
12316                .state
12317                .events
12318                .iter()
12319                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
12320            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
12321            runner.state.events
12322        );
12323
12324        // The cache is still held: a later reentry must retry the same
12325        // round's verification again — not leave it looking exactly as
12326        // untouched as the first blocked attempt, which is indistinguishable
12327        // from never having tried again at all.
12328        runner
12329            .stop_reviewing("round budget spent", &shell, &repo)
12330            .await
12331            .expect("stop_reviewing retry");
12332        assert_eq!(
12333            runner.state.reviews.len(),
12334            1,
12335            "no new round was started: {:?}",
12336            runner.state.reviews
12337        );
12338        let last = runner.state.reviews.last().expect("round record");
12339        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
12340        assert!(
12341            last.verified_at.expect("still known") > first_attempt_at,
12342            "a second reentry must be a fresh attempt, not a stale copy of the first"
12343        );
12344        assert_ne!(runner.state.status, RunStatus::Blocked);
12345
12346        held.release();
12347    }
12348
12349    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
12350    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
12351    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
12352    /// own top-of-function fast path (`review_conclusion`) correctly reads
12353    /// this shape as `None` rather than guessing `Blocked`, and the loop's
12354    /// own `for` range is empty once the round budget is spent, so
12355    /// `review_loop` must retry the check itself rather than silently doing
12356    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
12357    /// above exercises directly, but through `review_loop`'s own entry point
12358    /// this time, proving the wiring between the two rather than just the
12359    /// retry logic in isolation.
12360    #[tokio::test]
12361    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
12362        crate::run::pin_test_home();
12363        let home = crate::run::home();
12364
12365        let tmp = tempfile::tempdir().expect("tempdir");
12366        let repo = tmp.path().join("repo");
12367        std::fs::create_dir_all(&repo).unwrap();
12368        init_repo(&repo);
12369        let head = crate::git::rev_parse(&repo, "HEAD")
12370            .await
12371            .expect("rev-parse");
12372        let cache_dir = tmp.path().join("target");
12373
12374        let mut config = Config::default();
12375        config.verify.e2e = vec![format!(
12376            "CARGO_TARGET_DIR='{}' test -f README.md",
12377            cache_dir.display()
12378        )];
12379        config.graph.review_rounds = 1;
12380        config.graph.timeout_verify = Some(2);
12381
12382        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
12383        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
12384            .expect("no io error acquiring directly")
12385        {
12386            crate::cache::AcquireOutcome::Acquired(g) => g,
12387            crate::cache::AcquireOutcome::Busy(b) => {
12388                panic!("expected the direct acquire to win the lease first: {b:?}")
12389            }
12390        };
12391
12392        let mut state = RunState::new(
12393            repo.clone(),
12394            "main".to_owned(),
12395            head.clone(),
12396            "task".to_owned(),
12397            config,
12398        );
12399        state.candidates = vec![Candidate {
12400            index: 0,
12401            label: 'A',
12402            agent: "alpha".to_owned(),
12403            branch: "does-not-exist".to_owned(),
12404            worktree: repo.clone(),
12405            summary: String::new(),
12406            stat: String::new(),
12407            files: 0,
12408            commits: 0,
12409            empty: false,
12410            failed: None,
12411            verified_noop: None,
12412            duration_ms: 0,
12413            folded: false,
12414        }];
12415        state.tally = Some(Tally {
12416            first_choice: BTreeMap::from([('A', 1)]),
12417            borda: BTreeMap::new(),
12418            winner: 'A',
12419            rankings: 1,
12420            unanimous_initial: true,
12421            deliberated: false,
12422            changed_votes: 0,
12423            unanimous_final: true,
12424            tie_break: None,
12425            judges: 0,
12426            present: 0,
12427            quorum: 0,
12428            met_quorum: true,
12429            uncontested: Some("only candidate A produced a change".to_owned()),
12430        });
12431        // The exact shape a prior process's `stop_reviewing` would have left
12432        // on disk: the round budget's last round, a real attempt already
12433        // made and already resource-blocked.
12434        state.reviews = vec![ReviewRound {
12435            round: 1,
12436            head: head.clone(),
12437            verified_head: Some(head.clone()),
12438            verified_at: Some(jiff::Timestamp::now()),
12439            reviews: Vec::new(),
12440            e2e: vec![CommandOutcome {
12441                command: format!(
12442                    "CARGO_TARGET_DIR='{}' test -f README.md",
12443                    cache_dir.display()
12444                ),
12445                code: None,
12446                output_tail: "waiting for the shared build cache".to_owned(),
12447                duration_ms: 0,
12448                resource_blocked: true,
12449            }],
12450            fix: None,
12451            blocking: 1,
12452            answered: 1,
12453            expected: 1,
12454            clean: false,
12455            verify_retried: false,
12456            e2e_deferred: false,
12457            e2e_defer_reason: None,
12458            progressed: false,
12459            vote_split: false,
12460            reconsideration: Vec::new(),
12461            verdict: None,
12462        }];
12463
12464        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
12465        let mut runner = Runner {
12466            state,
12467            roles: ResolvedRoles {
12468                implementers: Vec::new(),
12469                judges: Vec::new(),
12470                reviewers: Vec::new(),
12471                fixer: None,
12472                conductor: conductor(),
12473                implementer_roster: Vec::new(),
12474                judge_roster: Vec::new(),
12475                reviewer_roster: Vec::new(),
12476            },
12477            sem: Arc::new(Semaphore::new(1)),
12478            pause: Pause::new(),
12479            interrupt: Pause::new(),
12480        };
12481
12482        // The lease is still held throughout, so this reentry's own retry is
12483        // also contended — proving `review_loop` actually tried again (not
12484        // that it happened to succeed) is what the timestamp comparison
12485        // below is for.
12486        runner.review_loop().await.expect("review_loop");
12487
12488        assert_eq!(
12489            runner.state.reviews.len(),
12490            1,
12491            "no new round was started on top of the unresolved one: {:?}",
12492            runner.state.reviews
12493        );
12494        let last = &runner.state.reviews[0];
12495        assert_eq!(
12496            last.e2e_status(),
12497            E2eStatus::ResourceBlocked,
12498            "still contended: {last:?}"
12499        );
12500        assert!(
12501            last.verified_at.expect("still known") > first_attempt_at,
12502            "review_loop must have actually retried the check, not left it exactly as found"
12503        );
12504        assert_ne!(
12505            runner.state.status,
12506            RunStatus::Blocked,
12507            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
12508            runner.state.status
12509        );
12510
12511        held.release();
12512    }
12513
12514    #[tokio::test]
12515    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
12516        crate::run::pin_test_home();
12517        let tmp = tempfile::tempdir().expect("tempdir");
12518        let repo = tmp.path().join("repo");
12519        std::fs::create_dir_all(&repo).unwrap();
12520        init_repo(&repo);
12521
12522        let mut config = Config::default();
12523        config.merge.mode = MergeMode::Pr;
12524        config.graph.land = true;
12525        config.graph.land_approval = false;
12526
12527        let mut state = RunState::new(
12528            repo.clone(),
12529            "main".to_owned(),
12530            "deadbeef".to_owned(),
12531            "task".to_owned(),
12532            config,
12533        );
12534        state.candidates = vec![Candidate {
12535            index: 0,
12536            label: 'A',
12537            agent: "alpha".to_owned(),
12538            branch: "does-not-exist".to_owned(),
12539            worktree: repo.clone(),
12540            summary: String::new(),
12541            stat: String::new(),
12542            files: 0,
12543            commits: 0,
12544            empty: false,
12545            failed: None,
12546            verified_noop: None,
12547            duration_ms: 0,
12548            folded: false,
12549        }];
12550        state.tally = Some(Tally {
12551            first_choice: BTreeMap::from([('A', 1)]),
12552            borda: BTreeMap::new(),
12553            winner: 'A',
12554            rankings: 1,
12555            unanimous_initial: true,
12556            deliberated: false,
12557            changed_votes: 0,
12558            unanimous_final: true,
12559            tie_break: None,
12560            judges: 0,
12561            present: 0,
12562            quorum: 0,
12563            met_quorum: true,
12564            uncontested: Some("only candidate A produced a change".to_owned()),
12565        });
12566        state.reviews = vec![ReviewRound {
12567            round: 1,
12568            head: "deadbeef".to_owned(),
12569            verified_head: None,
12570            verified_at: None,
12571            reviews: Vec::new(),
12572            e2e: Vec::new(),
12573            fix: None,
12574            blocking: 0,
12575            answered: 0,
12576            expected: 0,
12577            clean: true,
12578            verify_retried: false,
12579            e2e_deferred: false,
12580            e2e_defer_reason: None,
12581            progressed: false,
12582            vote_split: false,
12583            reconsideration: Vec::new(),
12584            verdict: None,
12585        }];
12586        state.gate = vec![CommandOutcome {
12587            command: "test".to_owned(),
12588            code: Some(0),
12589            output_tail: String::new(),
12590            duration_ms: 0,
12591            resource_blocked: false,
12592        }];
12593        state.gate_ran = true;
12594        // A first pass through `merge` already pushed and opened this pull
12595        // request; `status` is `Landing` because a previous call into `land`
12596        // parked or was interrupted before it reached a terminal outcome.
12597        state.status = RunStatus::Landing;
12598        state.merge = Some(MergeOutcome {
12599            mode: MergeMode::Pr,
12600            ok: true,
12601            detail: "https://example.invalid/x/y/pull/1".to_owned(),
12602            empty: false,
12603        });
12604
12605        // The Landing-resume shortcut calls `run_land` directly rather than
12606        // through `merge`, which is exactly the call site that used to skip
12607        // `settle_questions` - see the fixture below.
12608        ask_test_home();
12609        // A real resume loads a persisted run; driver ownership is checked
12610        // against that record before the Landing shortcut is entered.
12611        state.save().expect("save landing run");
12612        let store = ask::Questions::open();
12613        let q = ask_open_question(&store, &state.id);
12614
12615        let mut runner = Runner {
12616            state,
12617            roles: ResolvedRoles {
12618                implementers: Vec::new(),
12619                judges: Vec::new(),
12620                reviewers: Vec::new(),
12621                fixer: None,
12622                conductor: conductor(),
12623                implementer_roster: Vec::new(),
12624                judge_roster: Vec::new(),
12625                reviewer_roster: Vec::new(),
12626            },
12627            sem: Arc::new(Semaphore::new(1)),
12628            pause: Pause::new(),
12629            interrupt: Pause::new(),
12630        };
12631
12632        // `execute`, not `merge` directly: the Landing-resume shortcut lives
12633        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
12634        // exactly because `review_loop` would otherwise clobber the marker
12635        // first.
12636        runner.execute().await.expect("execute");
12637
12638        assert_eq!(
12639            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
12640            Some("https://example.invalid/x/y/pull/1"),
12641            "reentry must not push again or open a second pull request over the \
12642             one `land` is already watching"
12643        );
12644        assert_ne!(
12645            runner.state.status,
12646            RunStatus::Landing,
12647            "land could not actually reach the fake pull request, so it must \
12648             have given up rather than left the run silently parked forever"
12649        );
12650        // `land` could not reach the fake pull request, so it gave up into
12651        // `Blocked` - still resumable, so the question must not have been
12652        // swept just because this branch now also calls `settle_questions`.
12653        assert_eq!(runner.state.status, RunStatus::Blocked);
12654        assert!(
12655            store.get(&q.id).unwrap().status.open(),
12656            "Blocked is still alive; settle_questions must have been a no-op here"
12657        );
12658    }
12659
12660    fn state_with_round(round: ReviewRound) -> RunState {
12661        let mut s = RunState::new(
12662            PathBuf::from("/repo"),
12663            "main".to_owned(),
12664            "abc1234".to_owned(),
12665            "add retries".to_owned(),
12666            Config::default(),
12667        );
12668        s.reviews = vec![round];
12669        s
12670    }
12671
12672    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
12673        crate::verdict::Finding {
12674            id: id.to_owned(),
12675            severity,
12676            file: None,
12677            line: None,
12678            title: title.to_owned(),
12679            detail: String::new(),
12680        }
12681    }
12682
12683    #[test]
12684    fn pr_body_names_open_findings_and_declined_ones() {
12685        let round = ReviewRound {
12686            round: 2,
12687            head: "deadbee".to_owned(),
12688            verified_head: None,
12689            verified_at: None,
12690            reviews: vec![ReviewRecord {
12691                attempts: 0,
12692                reviewer: 1,
12693                agent: "alpha".to_owned(),
12694                summary: String::new(),
12695                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
12696                vote: None,
12697                failed: None,
12698                duration_ms: 0,
12699            }],
12700            e2e: vec![CommandOutcome {
12701                command: "cargo test".to_owned(),
12702                code: Some(0),
12703                output_tail: String::new(),
12704                duration_ms: 0,
12705                resource_blocked: false,
12706            }],
12707            verify_retried: false,
12708            e2e_deferred: false,
12709            e2e_defer_reason: None,
12710            fix: Some(FixRecord {
12711                agent: "alpha".to_owned(),
12712                addressed: Vec::new(),
12713                rejected: vec![crate::verdict::Rejection {
12714                    id: "R1-1-1".to_owned(),
12715                    why: "not reachable from any caller".to_owned(),
12716                }],
12717                notes: String::new(),
12718                committed: true,
12719                failed: None,
12720                duration_ms: 0,
12721                continuation: None,
12722            }),
12723            blocking: 0,
12724            answered: 1,
12725            expected: 1,
12726            clean: false,
12727            progressed: true,
12728            vote_split: false,
12729            reconsideration: Vec::new(),
12730            verdict: None,
12731        };
12732        let state = state_with_round(round);
12733        let body = pr_message(&state, 'A').body;
12734
12735        assert!(body.contains("add retries"), "the task must still be there");
12736        assert!(body.contains("R2-1-1"), "{body}");
12737        assert!(body.contains("unused import"), "{body}");
12738        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
12739        assert!(
12740            body.contains("not reachable from any caller"),
12741            "the reason it was declined: {body}"
12742        );
12743    }
12744
12745    #[test]
12746    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
12747        let round = ReviewRound {
12748            round: 1,
12749            head: "deadbee".to_owned(),
12750            verified_head: None,
12751            verified_at: None,
12752            reviews: vec![ReviewRecord {
12753                attempts: 0,
12754                reviewer: 1,
12755                agent: "alpha".to_owned(),
12756                summary: String::new(),
12757                findings: Vec::new(),
12758                vote: None,
12759                failed: None,
12760                duration_ms: 0,
12761            }],
12762            e2e: Vec::new(),
12763            verify_retried: false,
12764            e2e_deferred: false,
12765            e2e_defer_reason: None,
12766            fix: None,
12767            blocking: 0,
12768            answered: 1,
12769            expected: 1,
12770            clean: true,
12771            progressed: false,
12772            vote_split: false,
12773            reconsideration: Vec::new(),
12774            verdict: None,
12775        };
12776        let state = state_with_round(round);
12777        let body = pr_message(&state, 'A').body;
12778        assert!(!body.contains("Open review findings"), "{body}");
12779        assert!(!body.contains("Declined"), "{body}");
12780    }
12781
12782    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
12783        let mut state = RunState::new(
12784            PathBuf::from("/repo"),
12785            "main".to_owned(),
12786            "abc1234".to_owned(),
12787            instruction.to_owned(),
12788            Config::default(),
12789        );
12790        state.candidates.push(Candidate {
12791            index: 0,
12792            label: 'A',
12793            agent: "alpha".to_owned(),
12794            branch: "magi/x/A".to_owned(),
12795            worktree: PathBuf::from("/wt"),
12796            summary: summary.to_owned(),
12797            stat: String::new(),
12798            files: 1,
12799            commits: 1,
12800            empty: false,
12801            failed: None,
12802            verified_noop: None,
12803            folded: false,
12804            duration_ms: 0,
12805        });
12806        state
12807    }
12808
12809    fn review_state(subjects: &[&str]) -> RunState {
12810        let mut state = state_with_summary(
12811            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
12812            "",
12813        );
12814        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
12815        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
12816        state
12817    }
12818
12819    /// A branch rebased onto a main that moved past the recorded
12820    /// `base_commit` is titled from its own first commit, never main's.
12821    #[tokio::test]
12822    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
12823        ask_test_home();
12824        let tmp = tempfile::tempdir().unwrap();
12825        let repo = tmp.path().join("repo");
12826        std::fs::create_dir_all(&repo).unwrap();
12827        init_repo(&repo);
12828        let origin = tmp.path().join("origin.git");
12829        let g = |dir: &Path, args: &[&str]| {
12830            let out = std::process::Command::new("git")
12831                .args(args)
12832                .current_dir(dir)
12833                .quiet()
12834                .output()
12835                .expect("spawn git");
12836            assert!(
12837                out.status.success(),
12838                "git {args:?}: {}",
12839                String::from_utf8_lossy(&out.stderr)
12840            );
12841        };
12842        g(
12843            tmp.path(),
12844            &[
12845                "clone",
12846                "--bare",
12847                "-q",
12848                repo.to_str().unwrap(),
12849                origin.to_str().unwrap(),
12850            ],
12851        );
12852        g(
12853            &repo,
12854            &["remote", "add", "origin", origin.to_str().unwrap()],
12855        );
12856        let c1 = git::rev_parse(&repo, "main").await.unwrap();
12857
12858        // Main moves on; the branch is built on top of the new main.
12859        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
12860        g(&repo, &["add", "-A"]);
12861        g(
12862            &repo,
12863            &["commit", "-q", "-m", "chore(deps): update a crate"],
12864        );
12865        g(&repo, &["push", "-q", "origin", "main"]);
12866        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
12867        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
12868        g(&repo, &["add", "-A"]);
12869        g(
12870            &repo,
12871            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
12872        );
12873        g(&repo, &["checkout", "-q", "main"]);
12874
12875        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
12876        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
12877        // Without a readable tracking ref the recorded base's merge base is used.
12878        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
12879        assert_eq!(fallback, c1);
12880
12881        let mut state = review_state(&[
12882            "chore(deps): update a crate",
12883            "fix(daemon): apply a chosen action",
12884        ]);
12885        state.repo = repo.clone();
12886        state.base_branch = "main".to_owned();
12887        state.base_commit = c1;
12888        refresh_reviewed_commits(&mut state, "feat/own").await;
12889        assert_eq!(
12890            state.reviewed_commits,
12891            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
12892        );
12893        assert_eq!(
12894            review_title(&state).as_deref(),
12895            Some("fix(daemon): apply a chosen action")
12896        );
12897        assert_eq!(
12898            leaked_subjects(&state, "feat/own").await,
12899            Some(vec!["chore(deps): update a crate".to_owned()])
12900        );
12901        // A stale tracking ref is still used when the fetch fails, but the
12902        // leak list is withheld.
12903        state.config.merge.remote = "nowhere".to_owned();
12904        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
12905    }
12906
12907    #[test]
12908    fn pr_message_review_single_commit_uses_its_subject() {
12909        let state = review_state(&["feat(nats): per-role user"]);
12910        let m = pr_message(&state, 'A');
12911        assert_eq!(m.title, "feat(nats): per-role user");
12912        assert!(!m.body.contains("Review the work already"), "{}", m.body);
12913        assert!(m.body.contains("## Commits under review"), "{}", m.body);
12914    }
12915
12916    #[test]
12917    fn pr_message_review_multi_commit_takes_the_oldest() {
12918        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
12919        let m = pr_message(&state, 'A');
12920        assert_eq!(m.title, "feat: the change");
12921        for s in ["feat: the change", "fix: typo", "fix: again"] {
12922            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
12923        }
12924    }
12925
12926    #[test]
12927    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
12928        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
12929            let mut state = review_state(&[first, "fix: later fixup"]);
12930            state.candidates[0].branch = "機能/ログイン".to_owned();
12931            let m = pr_message(&state, 'A');
12932            assert!(
12933                m.title.starts_with("chore: land candidate A of run"),
12934                "{}",
12935                m.title
12936            );
12937        }
12938    }
12939
12940    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
12941        BranchFacts {
12942            commits: commits
12943                .iter()
12944                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
12945                .collect(),
12946            stat: stat.to_owned(),
12947        }
12948    }
12949
12950    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
12951        let mut r = review_round(true, 0, 1, 1, true, true);
12952        r.round = round;
12953        r.fix = notes.map(|n| FixRecord {
12954            agent: "fixer".to_owned(),
12955            addressed: Vec::new(),
12956            rejected: Vec::new(),
12957            notes: n.to_owned(),
12958            committed: true,
12959            failed: None,
12960            duration_ms: 0,
12961            continuation: None,
12962        });
12963        r
12964    }
12965
12966    #[test]
12967    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
12968        let state = review_state(&["ignored"]);
12969        let f = facts(
12970            &[
12971                (
12972                    "fix(login): resolve PATH on macOS",
12973                    "Login shells skip rc files.",
12974                ),
12975                ("fix: address review", ""),
12976            ],
12977            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
12978        );
12979        let m = pr_message_with(&state, 'A', Some(&f));
12980        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
12981        assert!(
12982            m.body
12983                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
12984            "{}",
12985            m.body
12986        );
12987        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
12988        assert!(m.body.contains("## Diff stat"), "{}", m.body);
12989        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
12990        for banned in [
12991            "Review the work already",
12992            "no task statement",
12993            "Original task",
12994        ] {
12995            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
12996        }
12997        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
12998    }
12999
13000    #[test]
13001    fn pr_message_review_truncates_a_huge_first_commit_body() {
13002        let state = review_state(&["ignored"]);
13003        let f = facts(
13004            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
13005            "s",
13006        );
13007        let m = pr_message_with(&state, 'A', Some(&f));
13008        assert!(m.body.len() < 30_000, "{}", m.body.len());
13009        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
13010        assert!(
13011            m.body.contains("- ... 1 more commit(s)"),
13012            "{}",
13013            m.body.len()
13014        );
13015        assert!(m.body.ends_with("magi:candidate-a\n"));
13016    }
13017
13018    #[test]
13019    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
13020        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
13021        assert_eq!(m.title, "feat: x");
13022        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
13023        assert!(!m.body.contains("Diff stat"), "{}", m.body);
13024        assert!(!m.body.contains("no task statement"), "{}", m.body);
13025    }
13026
13027    #[test]
13028    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
13029        let mut state = review_state(&["日本語の件名"]);
13030        state.candidates[0].branch = "fix/macos-login-path".to_owned();
13031        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
13032        state.candidates[0].branch = "機能/ログイン".to_owned();
13033        assert!(
13034            pr_message(&state, 'A')
13035                .title
13036                .starts_with("chore: land candidate A")
13037        );
13038    }
13039
13040    #[test]
13041    fn pr_message_review_fixes_survive_a_clean_final_round() {
13042        let mut state = review_state(&["feat: x"]);
13043        state.reviews = vec![
13044            round_with_notes(1, Some("handled the PATH case")),
13045            round_with_notes(2, None),
13046        ];
13047        let body = pr_message(&state, 'A').body;
13048        assert!(
13049            body.contains("## Review fixes\n\nhandled the PATH case\n"),
13050            "{body}"
13051        );
13052        assert!(!body.contains("### Round"), "{body}");
13053
13054        state.reviews = vec![
13055            round_with_notes(1, Some("first fix")),
13056            round_with_notes(2, Some("")),
13057            round_with_notes(3, Some("second fix")),
13058            round_with_notes(4, None),
13059        ];
13060        let body = pr_message(&state, 'A').body;
13061        assert!(body.contains("### Round 1\n\nfirst fix"), "{body}");
13062        assert!(body.contains("### Round 3\n\nsecond fix"), "{body}");
13063        assert!(!body.contains("### Round 2"), "{body}");
13064    }
13065
13066    const FINAL: &str = "abcdef1234567890";
13067
13068    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
13069        let mut r = round_with_notes(round, None);
13070        r.verified_head = verified.map(str::to_owned);
13071        r
13072    }
13073
13074    fn ok_gate(state: &mut RunState, head: Option<&str>) {
13075        state.gate = vec![CommandOutcome {
13076            command: "gate".to_owned(),
13077            code: Some(0),
13078            output_tail: String::new(),
13079            duration_ms: 0,
13080            resource_blocked: false,
13081        }];
13082        state.gate_ran = true;
13083        state.gate_head = head.map(str::to_owned);
13084    }
13085
13086    fn body_at(state: &RunState) -> String {
13087        pr_message_raw(state, 'A', None, crate::pr_outcome::FinalHead::At(FINAL)).body
13088    }
13089
13090    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
13091
13092    #[test]
13093    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
13094        let mut state = review_state(&["feat: x"]);
13095        state.reviews = vec![
13096            round_with_notes(1, Some(SANDBOXED)),
13097            passing_round(2, Some(FINAL)),
13098        ];
13099        ok_gate(&mut state, Some(FINAL));
13100        let body = body_at(&state);
13101        assert!(body.contains("Handled the PATH case."), "{body}");
13102        assert!(!body.contains("could not commit"), "{body}");
13103        assert!(!body.contains("could not run"), "{body}");
13104        assert!(
13105            body.contains("The fix is committed on the branch"),
13106            "{body}"
13107        );
13108        assert!(body.contains("2 statement(s)"), "{body}");
13109        assert!(
13110            body.contains("Tests (`verify.e2e`): passed on the final commit"),
13111            "{body}"
13112        );
13113        assert!(
13114            body.contains("Gate (`verify.gate`): passed on the final commit"),
13115            "{body}"
13116        );
13117        // The raw report is untouched in the record.
13118        assert!(
13119            state.reviews[0]
13120                .fix
13121                .as_ref()
13122                .unwrap()
13123                .notes
13124                .contains("could not commit")
13125        );
13126    }
13127
13128    #[test]
13129    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
13130        let mut state = review_state(&["feat: x"]);
13131        let mut r1 = round_with_notes(1, Some(SANDBOXED));
13132        r1.fix.as_mut().unwrap().committed = false;
13133        let mut r2 = passing_round(2, Some(FINAL));
13134        r2.e2e[0].code = Some(1);
13135        state.reviews = vec![r1, r2];
13136        let body = body_at(&state);
13137        assert!(body.contains("could not commit"), "{body}");
13138        assert!(body.contains("could not run"), "{body}");
13139        assert!(body.contains("This round produced no commit"), "{body}");
13140        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
13141        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
13142        assert!(!body.contains("passed"), "{body}");
13143    }
13144
13145    #[test]
13146    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
13147        let mut state = review_state(&["feat: x"]);
13148        state.reviews = vec![
13149            round_with_notes(1, Some(SANDBOXED)),
13150            passing_round(2, Some("0123456789abcdef")),
13151        ];
13152        ok_gate(&mut state, Some("0123456789abcdef"));
13153        let body = body_at(&state);
13154        assert!(body.contains("HTTP tests could not run"), "{body}");
13155        // The commit claim is still settled by the structured record.
13156        assert!(!body.contains("could not commit"), "{body}");
13157        assert!(
13158            body.contains("passed on an earlier commit (`0123456`)"),
13159            "{body}"
13160        );
13161        assert!(!body.contains("passed on the final commit"), "{body}");
13162
13163        // A gate recorded before `gate_head` existed is "not recorded".
13164        ok_gate(&mut state, None);
13165        let body = body_at(&state);
13166        assert!(
13167            body.contains("passed on a commit that was not recorded"),
13168            "{body}"
13169        );
13170    }
13171
13172    #[test]
13173    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
13174        let state = state_with_summary(
13175            "add retries to the client",
13176            "TITLE: feat: retries\n\nDid it.",
13177        );
13178        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
13179        assert_eq!(m.title, "feat: retries");
13180        assert!(
13181            m.body.contains("<summary>Original task</summary>"),
13182            "{}",
13183            m.body
13184        );
13185        assert!(!m.body.contains("Commits under review"), "{}", m.body);
13186        assert!(
13187            m.body
13188                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
13189            "{}",
13190            m.body
13191        );
13192    }
13193
13194    #[test]
13195    fn pr_message_review_bounds_a_long_english_subject() {
13196        let long = format!("feat: {}", "word ".repeat(100));
13197        let m = pr_message(&review_state(&[&long]), 'A');
13198        assert!(m.title.starts_with("feat: word"), "{}", m.title);
13199        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
13200    }
13201
13202    #[test]
13203    fn pr_message_implementation_run_is_unchanged_by_review_support() {
13204        let state = state_with_summary("add retries\n\ndetails", "- did some things");
13205        let m = pr_message(&state, 'A');
13206        assert_eq!(m.title, "add retries");
13207        assert!(m.body.contains("<summary>Original task</summary>"));
13208        assert!(!m.body.contains("Commits under review"));
13209        assert_eq!(landing_subject_source(&state), state.instruction);
13210    }
13211
13212    #[test]
13213    fn review_run_squash_subject_is_the_change_not_the_prompt() {
13214        let state = review_state(&["feat: the change", "fix: typo"]);
13215        let source = landing_subject_source(&state);
13216        assert_eq!(land::merge_subject("", &source), "feat: the change");
13217        assert_eq!(
13218            land::merge_subject("magi: candidate A (uncommitted work)", &source),
13219            "feat: the change"
13220        );
13221        // An operator's rename still wins.
13222        assert_eq!(
13223            land::merge_subject("feat: renamed by hand", &source),
13224            "feat: renamed by hand"
13225        );
13226        let blank = review_state(&["日本語"]);
13227        assert!(
13228            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
13229        );
13230    }
13231
13232    #[test]
13233    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
13234        let state = review_state(&["feat: the change"]);
13235        let source = landing_subject_source(&state);
13236        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
13237        assert_eq!(
13238            land::merge_subject(landing_title(&state, old), &source),
13239            "feat: the change"
13240        );
13241        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
13242        let task = state_with_summary("add retries", "");
13243        assert_eq!(landing_title(&task, old), old);
13244    }
13245
13246    #[test]
13247    fn pr_message_describes_the_change_not_the_task() {
13248        let state = state_with_summary(
13249            "今回やってほしいこと: results projector を直す",
13250            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
13251        );
13252        let m = pr_message(&state, 'A');
13253        assert_eq!(m.title, "fix(web): batch the runs list reads");
13254        assert!(
13255            m.body.starts_with("## Summary\n\n- reads run.json once"),
13256            "{}",
13257            m.body
13258        );
13259        assert!(!m.body.contains("TITLE:"), "{}", m.body);
13260        let task_at = m.body.find("今回やってほしいこと").unwrap();
13261        let details_at = m.body.find("<details>").unwrap();
13262        assert!(
13263            details_at < task_at,
13264            "the task lives inside <details>: {}",
13265            m.body
13266        );
13267        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
13268        assert!(m.body.contains("magi:candidate-a"));
13269    }
13270
13271    #[test]
13272    fn pr_message_falls_back_to_the_task_without_a_title_line() {
13273        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
13274        let m = pr_message(&state, 'A');
13275        assert_eq!(m.title, "add retries");
13276        assert!(
13277            m.body.contains("## Summary\n\n- did some things"),
13278            "{}",
13279            m.body
13280        );
13281
13282        let none = RunState::new(
13283            PathBuf::from("/repo"),
13284            "main".to_owned(),
13285            "abc1234".to_owned(),
13286            "add retries".to_owned(),
13287            Config::default(),
13288        );
13289        let m = pr_message(&none, 'A');
13290        assert_eq!(m.title, "add retries");
13291        assert!(!m.body.contains("## Summary"), "{}", m.body);
13292    }
13293
13294    #[test]
13295    fn pr_message_refuses_the_candidate_commit_subject() {
13296        for bad in [
13297            "TITLE: magi: candidate A (uncommitted work)",
13298            "TITLE: chore: stuff (uncommitted work)",
13299            "TITLE:   ",
13300        ] {
13301            let state = state_with_summary("add retries", bad);
13302            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
13303        }
13304    }
13305
13306    #[test]
13307    fn pr_message_bounds_a_very_long_task_and_title() {
13308        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
13309        let state = state_with_summary(&long, "- nothing");
13310        let m = pr_message(&state, 'A');
13311        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
13312        assert!(!m.title.contains('\n'));
13313
13314        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
13315        let m = pr_message(&state, 'A');
13316        assert!(m.title.starts_with("feat: "));
13317        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
13318        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
13319    }
13320
13321    fn long_title_of(instruction: &str) -> String {
13322        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
13323    }
13324
13325    #[test]
13326    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
13327        let first = "Make the landing path keep a readable title for long tasks";
13328        let line = format!(
13329            "{first}. {}",
13330            "And then keep going with more words ".repeat(20)
13331        );
13332        let t = long_title_of(&line);
13333        assert_eq!(t, first);
13334        assert!(!t.starts_with("chore: land"));
13335    }
13336
13337    #[test]
13338    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
13339        let line = "word ".repeat(200);
13340        let t = long_title_of(&line);
13341        assert!(t.ends_with("word..."), "{t}");
13342        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
13343    }
13344
13345    #[test]
13346    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
13347        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
13348            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
13349        }
13350    }
13351
13352    #[test]
13353    fn pr_message_title_limit_is_exact() {
13354        let at = "a".repeat(PR_TITLE_MAX);
13355        assert_eq!(long_title_of(&at), at);
13356        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
13357        assert!(over.ends_with("..."), "{over}");
13358        assert_eq!(over.chars().count(), PR_TITLE_MAX);
13359    }
13360
13361    #[test]
13362    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
13363        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
13364        let t = long_title_of(&line);
13365        assert!(t.ends_with("..."), "{t}");
13366        assert!(t.is_ascii(), "{t}");
13367    }
13368
13369    #[test]
13370    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
13371        let line = format!(
13372            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
13373            "plus more filler words ".repeat(20)
13374        );
13375        let t = long_title_of(&line);
13376        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
13377    }
13378
13379    #[test]
13380    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
13381        let line = format!("\n\n# {}", "title words ".repeat(40));
13382        let state = state_with_summary(&line, "- nothing");
13383        let m = pr_message(&state, 'A');
13384        assert!(m.title.starts_with("title words"), "{}", m.title);
13385        assert_eq!(
13386            land::merge_subject(&m.title, &landing_subject_source(&state)),
13387            m.title
13388        );
13389        // An operator's rename wins untouched.
13390        assert_eq!(
13391            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
13392            "feat: renamed by hand"
13393        );
13394    }
13395
13396    #[tokio::test]
13397    async fn github_text_rewrite_is_bounded_and_falls_back() {
13398        crate::run::pin_test_home();
13399        for (reply, accepted) in [
13400            (
13401                "TITLE: fix: retries\nAdd retries for failed requests.",
13402                true,
13403            ),
13404            (
13405                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
13406                false,
13407            ),
13408            (
13409                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
13410                true,
13411            ),
13412            (
13413                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
13414                true,
13415            ),
13416            (
13417                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
13418                true,
13419            ),
13420            (
13421                "TITLE: fix: retries\n日本語の説明です。 token=secret",
13422                false,
13423            ),
13424            ("TITLE: fix: retries\n", false),
13425        ] {
13426            let dir = tempfile::tempdir().unwrap();
13427            let mut runner = runner_at(RunStatus::Gating);
13428            runner.state = state_with_summary(
13429                "add retries",
13430                "TITLE: fix: retries\n日本語の説明を書きます。",
13431            );
13432            runner.state.repo = dir.path().to_owned();
13433            runner.state.candidates[0].worktree = dir.path().to_owned();
13434            let mut author = spec("alpha");
13435            author.command = vec![
13436                "sh".into(),
13437                "-c".into(),
13438                "printf '%s' \"$REWRITE_REPLY\"".into(),
13439            ];
13440            author.env.insert("REWRITE_REPLY".into(), reply.into());
13441            runner.state.config.agents = vec![author];
13442            let winner = runner.state.candidates[0].clone();
13443            let first = runner
13444                .guarded_pr_message(&winner, None, true)
13445                .await
13446                .unwrap();
13447            let message = if accepted {
13448                first.expect("a passing rewrite never asks")
13449            } else {
13450                // Withheld: the run parks on a question and posts nothing yet.
13451                assert!(first.is_none());
13452                assert!(runner.state.parked);
13453                let store = ask::Questions::open();
13454                let asked: Vec<_> = store
13455                    .list()
13456                    .into_iter()
13457                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
13458                    .collect();
13459                assert_eq!(asked.len(), 1);
13460                assert!(!asked[0].detail.contains("secret"));
13461                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
13462                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
13463                assert_eq!(
13464                    asked[0]
13465                        .choices
13466                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
13467                    rewrite.is_some()
13468                );
13469                // Silence: the question lapses and the neutral text is used,
13470                // without a second rewrite call or a second question.
13471                store
13472                    .update(&asked[0].id, |q| {
13473                        q.abandon("test");
13474                        Ok(())
13475                    })
13476                    .unwrap();
13477                let again = runner
13478                    .guarded_pr_message(&winner, None, true)
13479                    .await
13480                    .unwrap()
13481                    .expect("an abandoned question falls back");
13482                let n = store
13483                    .list()
13484                    .into_iter()
13485                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
13486                    .count();
13487                assert_eq!(n, 1, "asked once per rejected text");
13488                again
13489            };
13490            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
13491            assert!(!message.body.contains("secret"));
13492            assert!(!message.body.contains("prefix"));
13493            assert!(!message.title.contains("secret"));
13494            assert_eq!(
13495                message.body.contains("Add retries for failed requests."),
13496                accepted
13497            );
13498            assert_eq!(
13499                runner.state.seats["github-text-rewrite"].turns, 1,
13500                "only one rewrite invocation"
13501            );
13502            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
13503            if accepted {
13504                assert!(runner.state.github_text.is_none());
13505                assert!(!runner.state.parked);
13506                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
13507                assert!(
13508                    message
13509                        .body
13510                        .contains(&format!("magi:run/{}", runner.state.id))
13511                );
13512            }
13513            if !accepted {
13514                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
13515                assert!(
13516                    message
13517                        .body
13518                        .contains(&format!("magi:run/{}", runner.state.id))
13519                );
13520            }
13521        }
13522    }
13523
13524    fn scripted(id: &str, reply: &str) -> AgentSpec {
13525        let mut a = spec(id);
13526        a.command = vec![
13527            "sh".into(),
13528            "-c".into(),
13529            "printf '%s' \"$REWRITE_REPLY\"".into(),
13530        ];
13531        a.env.insert("REWRITE_REPLY".into(), reply.into());
13532        a
13533    }
13534
13535    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
13536    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
13537
13538    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
13539        let mut runner = runner_at(RunStatus::Gating);
13540        runner.state = state_with_summary("add retries", JA_SUMMARY);
13541        if review_only {
13542            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
13543        }
13544        runner.state.repo = dir.to_owned();
13545        runner.state.candidates[0].worktree = dir.to_owned();
13546        runner.state.config.agents = agents;
13547        runner
13548    }
13549
13550    fn gate_questions(runner: &Runner) -> usize {
13551        ask::Questions::open()
13552            .list()
13553            .into_iter()
13554            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
13555            .count()
13556    }
13557
13558    #[tokio::test]
13559    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
13560        crate::run::pin_test_home();
13561        let dir = tempfile::tempdir().unwrap();
13562        let mut runner = rewrite_runner(
13563            dir.path(),
13564            true,
13565            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
13566        );
13567        runner.state.config.roles.language_judge =
13568            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
13569        let winner = runner.state.candidates[0].clone();
13570        let msg = runner
13571            .guarded_pr_message(&winner, None, true)
13572            .await
13573            .unwrap()
13574            .expect("an English rewrite posts without asking");
13575        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
13576        assert!(msg.body.contains("<details>kept</details>"));
13577        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
13578        assert_eq!(gate_questions(&runner), 0);
13579        assert!(runner.state.github_text.is_none());
13580        assert!(!runner.state.seats.contains_key("impl-A"));
13581        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
13582    }
13583
13584    #[tokio::test]
13585    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
13586        crate::run::pin_test_home();
13587        let dir = tempfile::tempdir().unwrap();
13588        let mut runner = rewrite_runner(
13589            dir.path(),
13590            false,
13591            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
13592        );
13593        runner.state.config.roles.language_judge =
13594            Some(crate::config::AgentChoice::One("beta".into()));
13595        runner.state.seats.insert(
13596            "impl-A".into(),
13597            SeatState::new("impl-A", "alpha", runner.state.seed),
13598        );
13599        let winner = runner.state.candidates[0].clone();
13600        let msg = runner
13601            .guarded_pr_message(&winner, None, true)
13602            .await
13603            .unwrap()
13604            .expect("rewritten");
13605        assert!(msg.body.contains("<details>kept</details>"));
13606        assert_eq!(runner.state.seats["impl-A"].turns, 0);
13607        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
13608        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
13609    }
13610
13611    #[tokio::test]
13612    async fn rewrite_reuses_a_resumable_implementer_session() {
13613        crate::run::pin_test_home();
13614        let dir = tempfile::tempdir().unwrap();
13615        let mut runner = rewrite_runner(
13616            dir.path(),
13617            false,
13618            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
13619        );
13620        runner.state.config.roles.language_judge =
13621            Some(crate::config::AgentChoice::One("beta".into()));
13622        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
13623        seat.turns = 1;
13624        runner.state.seats.insert("impl-A".into(), seat);
13625        let winner = runner.state.candidates[0].clone();
13626        runner
13627            .guarded_pr_message(&winner, None, true)
13628            .await
13629            .unwrap()
13630            .expect("rewritten");
13631        assert_eq!(runner.state.seats["impl-A"].turns, 2);
13632        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
13633    }
13634
13635    #[tokio::test]
13636    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
13637        crate::run::pin_test_home();
13638        let dir = tempfile::tempdir().unwrap();
13639        let mut broken = spec("bad");
13640        broken.command = vec!["false".into()];
13641        let mut runner = rewrite_runner(
13642            dir.path(),
13643            true,
13644            vec![
13645                broken,
13646                scripted("noformat", "Add retries."),
13647                scripted("good", EN_REWRITE),
13648            ],
13649        );
13650        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
13651            "bad".into(),
13652            "noformat".into(),
13653            "good".into(),
13654        ]));
13655        let winner = runner.state.candidates[0].clone();
13656        let msg = runner
13657            .guarded_pr_message(&winner, None, true)
13658            .await
13659            .unwrap()
13660            .expect("the third agent rewrites");
13661        assert!(msg.body.contains("`src/net.rs`"));
13662        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
13663        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
13664    }
13665
13666    #[tokio::test]
13667    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
13668        crate::run::pin_test_home();
13669        // (agents, role): still Japanese; no TITLE; nothing resolvable.
13670        for (agents, role, saved) in [
13671            (
13672                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
13673                Some("beta"),
13674                true,
13675            ),
13676            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
13677            (Vec::new(), None, false),
13678        ] {
13679            let dir = tempfile::tempdir().unwrap();
13680            let mut runner = rewrite_runner(dir.path(), true, agents);
13681            runner.state.config.roles.language_judge =
13682                role.map(|r| crate::config::AgentChoice::One(r.into()));
13683            let winner = runner.state.candidates[0].clone();
13684            let first = runner
13685                .guarded_pr_message(&winner, None, true)
13686                .await
13687                .unwrap();
13688            assert!(first.is_none(), "withheld text parks the run");
13689            assert!(runner.state.parked);
13690            assert_eq!(gate_questions(&runner), 1);
13691            let g = runner.state.github_text.as_ref().unwrap();
13692            assert_eq!(g.rewrite.is_some(), saved);
13693            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
13694        }
13695    }
13696
13697    #[tokio::test]
13698    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
13699        crate::run::pin_test_home();
13700        let dir = tempfile::tempdir().unwrap();
13701        let mut runner = runner_at(RunStatus::Gating);
13702        runner.state =
13703            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
13704        runner.state.repo = dir.path().to_owned();
13705        runner.state.candidates[0].worktree = dir.path().to_owned();
13706        let mut author = spec("alpha");
13707        author.command = vec![
13708            "sh".into(),
13709            "-c".into(),
13710            "printf '%s' \"$REWRITE_REPLY\"".into(),
13711        ];
13712        author.env.insert(
13713            "REWRITE_REPLY".into(),
13714            format!(
13715                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
13716                "English sentence. ".repeat(600)
13717            ),
13718        );
13719        runner.state.config.agents = vec![author];
13720        let winner = runner.state.candidates[0].clone();
13721        assert!(
13722            runner
13723                .guarded_pr_message(&winner, None, true)
13724                .await
13725                .unwrap()
13726                .is_none()
13727        );
13728        let record = runner.state.github_text.clone().unwrap();
13729        let rewrite = record.rewrite.clone().unwrap();
13730        assert_eq!(rewrite.failed, ["body-language"]);
13731        assert!(
13732            rewrite
13733                .body
13734                .ends_with(&format!("magi:run/{}", runner.state.id))
13735        );
13736        let store = ask::Questions::open();
13737        let q = store.get(record.question.as_ref().unwrap()).unwrap();
13738        assert!(
13739            q.choices
13740                .contains(&crate::github_text::USE_REWRITE.to_owned())
13741        );
13742        for label in [
13743            "ORIGINAL",
13744            "FALLBACK",
13745            "REWRITE",
13746            "Rules still failed: body-language",
13747            "truncated",
13748        ] {
13749            assert!(q.detail.contains(label), "{label}");
13750        }
13751        let artifact = std::fs::read_to_string(crate::run::artifact_path(
13752            &runner.state,
13753            &crate::github_text::artifact_name(&record.fingerprint),
13754        ))
13755        .unwrap();
13756        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
13757        // Restart from serialized state, then retry after a bad owner title.
13758        runner.state =
13759            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
13760        store
13761            .update(&q.id, |q| {
13762                q.say("まだ日本語")?;
13763                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
13764            })
13765            .unwrap();
13766        assert!(
13767            runner
13768                .guarded_pr_message(&winner, None, true)
13769                .await
13770                .unwrap()
13771                .is_none()
13772        );
13773        let retry = runner.state.github_text.as_ref().unwrap();
13774        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
13775        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
13776        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
13777        store
13778            .update(&q.id, |q| {
13779                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
13780            })
13781            .unwrap();
13782        let message = runner
13783            .guarded_pr_message(&winner, None, true)
13784            .await
13785            .unwrap()
13786            .unwrap();
13787        assert_eq!(message.title, rewrite.title);
13788        assert_eq!(message.body, rewrite.body);
13789        assert_eq!(
13790            runner.state.github_text.as_ref().unwrap().choice,
13791            Some(GithubTextChoice::Rewrite)
13792        );
13793        let again = runner
13794            .guarded_pr_message(&winner, None, true)
13795            .await
13796            .unwrap()
13797            .unwrap();
13798        assert_eq!(again.title, message.title);
13799        assert_eq!(again.body, message.body);
13800        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
13801        assert!(
13802            !runner
13803                .state
13804                .events
13805                .iter()
13806                .any(|e| e.message.contains(&rewrite.title)
13807                    || e.message.contains("English sentence."))
13808        );
13809    }
13810
13811    #[tokio::test]
13812    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
13813        crate::run::pin_test_home();
13814        let dir = tempfile::tempdir().unwrap();
13815        let mut runner = runner_at(RunStatus::Gating);
13816        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
13817        runner.state.repo = dir.path().to_owned();
13818        runner.state.candidates[0].worktree = dir.path().to_owned();
13819        let winner = runner.state.candidates[0].clone();
13820        assert!(
13821            runner
13822                .guarded_pr_message(&winner, None, true)
13823                .await
13824                .unwrap()
13825                .is_none()
13826        );
13827        let record = runner.state.github_text.as_ref().unwrap();
13828        assert!(record.rewrite.is_none());
13829        // Records written before the additive field deserialize with no rewrite.
13830        let mut old = serde_json::to_value(record).unwrap();
13831        old.as_object_mut().unwrap().remove("rewrite");
13832        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
13833        assert!(old.rewrite.is_none());
13834        let id = record.question.clone().unwrap();
13835        ask::Questions::open()
13836            .update(&id, |q| {
13837                q.choices.push(crate::github_text::USE_REWRITE.into());
13838                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
13839            })
13840            .unwrap();
13841        let message = runner
13842            .guarded_pr_message(&winner, None, true)
13843            .await
13844            .unwrap()
13845            .unwrap();
13846        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
13847        assert_eq!(
13848            runner.state.github_text.as_ref().unwrap().choice,
13849            Some(GithubTextChoice::Fallback)
13850        );
13851    }
13852
13853    #[tokio::test]
13854    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
13855        crate::run::pin_test_home();
13856        let dir = tempfile::tempdir().unwrap();
13857        let mut runner = runner_at(RunStatus::Gating);
13858        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
13859        runner.state.repo = dir.path().to_owned();
13860        runner.state.candidates[0].worktree = dir.path().to_owned();
13861        let winner = runner.state.candidates[0].clone();
13862        assert!(
13863            runner
13864                .guarded_pr_message(&winner, None, true)
13865                .await
13866                .unwrap()
13867                .is_none()
13868        );
13869        let store = ask::Questions::open();
13870        let find = |run: &str| {
13871            store
13872                .list()
13873                .into_iter()
13874                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
13875                .unwrap()
13876        };
13877        let run = runner.state.id.clone();
13878        let q = find(&run);
13879        let fp = runner
13880            .state
13881            .github_text
13882            .as_ref()
13883            .unwrap()
13884            .fingerprint
13885            .clone();
13886        let name = crate::github_text::artifact_name(&fp);
13887        let path = crate::run::artifact_path(&runner.state, &name);
13888        assert!(q.detail.contains(&path.display().to_string()));
13889        assert!(q.detail.contains("title-language: title"));
13890        let saved = std::fs::read_to_string(&path).expect("artifact written");
13891        assert!(saved.contains("日本語のタイトル"));
13892        assert!(q.cwd.is_none());
13893        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
13894        // A bad replacement is refused and asked about once more.
13895        let say = |id: &str, text: &str, choice: &str| {
13896            store
13897                .update(id, |q| {
13898                    q.thread.push(ask::Turn {
13899                        who: ask::Who::Operator,
13900                        body: text.to_owned(),
13901                        at: jiff::Timestamp::now(),
13902                        note: None,
13903                    });
13904                    q.answer(ask::Answer::Choice(choice.to_owned()))
13905                })
13906                .unwrap();
13907        };
13908        say(&q.id, "まだ日本語", "use my text");
13909        assert!(
13910            runner
13911                .guarded_pr_message(&winner, None, true)
13912                .await
13913                .unwrap()
13914                .is_none()
13915        );
13916        let second = find(&run);
13917        assert_ne!(second.id, q.id);
13918        assert!(second.detail.contains("did not pass"));
13919        // The retry still compares the original text; the rejected
13920        // replacement is named by category only.
13921        assert!(second.detail.contains("日本語のタイトル"));
13922        assert!(!second.detail.contains("まだ日本語"));
13923        assert!(second.detail.contains(&path.display().to_string()));
13924        say(&second.id, "fix: retry failed requests", "use my text");
13925        let message = runner
13926            .guarded_pr_message(&winner, None, true)
13927            .await
13928            .unwrap()
13929            .expect("a vetted replacement is posted");
13930        assert_eq!(message.title, "fix: retry failed requests");
13931        assert_eq!(
13932            runner
13933                .state
13934                .github_text
13935                .as_ref()
13936                .unwrap()
13937                .chosen_title
13938                .as_deref(),
13939            Some("fix: retry failed requests")
13940        );
13941        assert!(runner.state.github_text.as_ref().unwrap().resolved);
13942        // Resumed again: the saved decision is applied, nothing is re-asked.
13943        let again = runner
13944            .guarded_pr_message(&winner, None, true)
13945            .await
13946            .unwrap()
13947            .unwrap();
13948        assert_eq!(again.title, message.title);
13949        assert_eq!(
13950            store
13951                .list()
13952                .into_iter()
13953                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
13954                .count(),
13955            2
13956        );
13957    }
13958
13959    #[test]
13960    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
13961        // What magi itself writes stays English under any configured language,
13962        // so a future localisation of these headings fails here. (The agents'
13963        // own text is also checked by the posting gate.)
13964        let mut state = state_with_summary(
13965            "add retries",
13966            "TITLE: fix(web): batch reads\n- reads run.json once",
13967        );
13968        state.config.graph.language = "ja".to_owned();
13969        let m = pr_message(&state, 'A');
13970        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
13971        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
13972
13973        // The task is the operator's own text: it goes in untouched, and the
13974        // fallback title (no summary) may be in its language too.
13975        let task = "今回やってほしいこと: results projector を直す";
13976        let mut state = state_with_summary(task, "- no title line");
13977        state.config.graph.language = "ja".to_owned();
13978        let m = pr_message(&state, 'A');
13979        assert_eq!(
13980            m.title,
13981            format!("chore: land candidate A of run {}", state.id)
13982        );
13983        assert!(
13984            m.body.contains(&format!(
13985                "<summary>Original task</summary>\n\n{task}\n\n</details>"
13986            )),
13987            "{}",
13988            m.body
13989        );
13990    }
13991
13992    #[test]
13993    fn pr_message_scrubs_home_paths_and_addresses() {
13994        let state = state_with_summary(
13995            "fix it in /Users/someone/src/x",
13996            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
13997        );
13998        let m = pr_message(&state, 'A');
13999        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
14000            assert!(!m.body.contains(leak), "{}", m.body);
14001        }
14002        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
14003    }
14004
14005    #[test]
14006    fn pr_message_survives_a_task_that_closes_details() {
14007        let state = state_with_summary("a </details> b", "TITLE: fix: x");
14008        let m = pr_message(&state, 'A');
14009        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
14010    }
14011
14012    #[test]
14013    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
14014        let cmd = manual_merge_command(
14015            MergeStyle::Squash,
14016            Path::new("/repo"),
14017            "b",
14018            "fix: \"quoted\" $(x) `y`\n\nbody",
14019        );
14020        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
14021    }
14022
14023    #[test]
14024    fn manual_merge_command_matches_the_configured_style() {
14025        let repo = Path::new("/repo");
14026        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
14027
14028        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
14029        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
14030
14031        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
14032        assert_eq!(
14033            squash,
14034            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
14035             \"Merge magi run 0832 (candidate A)\""
14036        );
14037
14038        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
14039        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
14040    }
14041
14042    #[test]
14043    fn a_nudge_gets_a_quarter_of_the_budget() {
14044        // The judge and implement budgets magi ships with.
14045        assert_eq!(retry_budget(secs(1200), true), secs(300));
14046        assert_eq!(retry_budget(secs(3600), true), secs(900));
14047    }
14048
14049    #[test]
14050    fn a_resent_prompt_keeps_the_whole_budget() {
14051        // The seat kept no context, so the retry is the original job again and
14052        // shortening it would only guarantee a second failure.
14053        assert_eq!(retry_budget(secs(1200), false), secs(1200));
14054        assert_eq!(retry_budget(secs(60), false), secs(60));
14055    }
14056
14057    #[test]
14058    fn the_floor_never_exceeds_the_original_budget() {
14059        // A short configured timeout must not be *raised* by the floor: the
14060        // operator asked for a bound, and a retry may not outlast the attempt
14061        // it is retrying.
14062        assert_eq!(retry_budget(secs(60), true), secs(60));
14063        assert_eq!(retry_budget(secs(480), true), secs(120));
14064        assert_eq!(retry_budget(secs(0), true), secs(0));
14065    }
14066
14067    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
14068        agent::CommandEvidence {
14069            id: "item1".to_owned(),
14070            description: "cargo test".to_owned(),
14071            exit_code,
14072            result_summary: String::new(),
14073            source: "codex".to_owned(),
14074        }
14075    }
14076
14077    #[test]
14078    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
14079        // No evidence is not the same fact as unconfirmed evidence: a
14080        // backend with no adapter, or a reply that ran no commands at all,
14081        // must not be misread as carrying a dangling job.
14082        assert!(!has_unconfirmed_command(&[]));
14083    }
14084
14085    #[test]
14086    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
14087        // Deliberately not a check on the exit code's *value*: a fixer
14088        // legitimately runs something that fails mid-iteration before it
14089        // succeeds, and that must never by itself reopen a valid report.
14090        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
14091        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
14092        assert!(!has_unconfirmed_command(&[
14093            evidence(Some(0)),
14094            evidence(Some(101))
14095        ]));
14096    }
14097
14098    #[test]
14099    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
14100        assert!(has_unconfirmed_command(&[
14101            evidence(Some(0)),
14102            evidence(None)
14103        ]));
14104    }
14105
14106    #[test]
14107    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
14108        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
14109        assert_eq!(
14110            verified_noop_claim(true, &[], text).as_deref(),
14111            Some("already fixed by b32cfc4, on main.")
14112        );
14113    }
14114
14115    #[test]
14116    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
14117        // A timeout or a bad exit code reads as the ordinary loss it is,
14118        // whatever the reply's own prose claims.
14119        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
14120        assert!(verified_noop_claim(false, &[], text).is_none());
14121    }
14122
14123    #[test]
14124    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
14125        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
14126        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
14127        // A confirmed command alongside the marker is fine.
14128        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
14129    }
14130
14131    #[test]
14132    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
14133        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
14134    }
14135
14136    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
14137    /// pair, in order, labelled A, B, C, ...
14138    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
14139        runner.state.candidates = shape
14140            .iter()
14141            .enumerate()
14142            .map(|(i, &(empty, verified))| Candidate {
14143                index: i,
14144                label: (b'A' + i as u8) as char,
14145                agent: "sonnet".to_owned(),
14146                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
14147                worktree: PathBuf::from(format!("/wt/{i}")),
14148                summary: String::new(),
14149                stat: String::new(),
14150                files: 0,
14151                commits: 0,
14152                empty,
14153                failed: None,
14154                verified_noop: verified.map(str::to_owned),
14155                duration_ms: 0,
14156                folded: false,
14157            })
14158            .collect();
14159    }
14160
14161    #[test]
14162    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
14163        ask_test_home();
14164        let mut runner = runner_at(RunStatus::Implementing);
14165        set_candidates(
14166            &mut runner,
14167            &[
14168                (true, Some("already on main at b32cfc4")),
14169                (true, Some("same fix, see the existing test")),
14170            ],
14171        );
14172
14173        runner
14174            .after_implement()
14175            .expect("a verified no-op is not an error");
14176
14177        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
14178    }
14179
14180    #[test]
14181    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
14182        ask_test_home();
14183        let mut runner = runner_at(RunStatus::Implementing);
14184        // Candidate A declares a verified no-op; candidate B simply wrote
14185        // nothing and said nothing about why. One candidate's claim is not
14186        // the whole run's agreement.
14187        set_candidates(
14188            &mut runner,
14189            &[(true, Some("already on main at b32cfc4")), (true, None)],
14190        );
14191
14192        let err = runner
14193            .after_implement()
14194            .expect_err("an unverified empty candidate must still fail the run");
14195
14196        assert!(
14197            err.to_string().contains("no candidate produced a change"),
14198            "{err}"
14199        );
14200        assert_eq!(runner.state.status, RunStatus::Failed);
14201    }
14202
14203    #[test]
14204    fn after_implement_still_fails_an_ordinary_all_empty_run() {
14205        ask_test_home();
14206        let mut runner = runner_at(RunStatus::Implementing);
14207        set_candidates(&mut runner, &[(true, None), (true, None)]);
14208
14209        let err = runner
14210            .after_implement()
14211            .expect_err("no candidate declared anything; this is an ordinary failure");
14212
14213        assert!(
14214            err.to_string().contains("no candidate produced a change"),
14215            "{err}"
14216        );
14217        assert_eq!(runner.state.status, RunStatus::Failed);
14218    }
14219}
14220
14221/// A graft rollback whose restore could not be confirmed (see
14222/// [`Runner::graft`]).
14223#[derive(Debug)]
14224struct GraftRestoreFailed(String);
14225
14226impl std::fmt::Display for GraftRestoreFailed {
14227    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
14228        write!(
14229            f,
14230            "could not restore the winner to its pre-graft commit: {}",
14231            self.0
14232        )
14233    }
14234}
14235
14236impl std::error::Error for GraftRestoreFailed {}
14237
14238/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
14239/// not the agent: they never draw on an agent's own timeout, and running out of
14240/// them never reads as an agent failure.
14241#[cfg(test)]
14242mod resource_wait_tests {
14243    use super::*;
14244    use std::collections::BTreeMap;
14245
14246    fn sh(id: &str, script: &str) -> AgentSpec {
14247        AgentSpec {
14248            id: id.to_owned(),
14249            kind: crate::config::AgentKind::Command,
14250            model: None,
14251            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
14252            extra_args: Vec::new(),
14253            env: BTreeMap::new(),
14254            prompt_delivery: None,
14255        }
14256    }
14257
14258    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
14259        SeatJob {
14260            seat: SeatState::new("impl-A", &spec.id, 1),
14261            spec,
14262            cwd: dir.to_path_buf(),
14263            prompt: "do it".to_owned(),
14264            timeout,
14265            allow_write: true,
14266            sessions: false,
14267            artifacts: dir.join("art"),
14268            stem: "impl-A".to_owned(),
14269            handover: None,
14270        }
14271    }
14272
14273    fn state(resource_wait_secs: u64) -> RunState {
14274        let mut config = Config::default();
14275        config.graph.timeout_resource_wait = resource_wait_secs;
14276        RunState::new(
14277            PathBuf::from("/repo"),
14278            "main".to_owned(),
14279            "abc1234".to_owned(),
14280            "task".to_owned(),
14281            config,
14282        )
14283    }
14284
14285    fn hold(cache: &Path) -> crate::cache::Guard {
14286        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
14287        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
14288            crate::cache::AcquireOutcome::Acquired(g) => g,
14289            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
14290        }
14291    }
14292
14293    async fn run(
14294        jobs: Vec<SeatJob>,
14295        sem: Arc<Semaphore>,
14296        cache: Option<&Path>,
14297        st: &mut RunState,
14298    ) -> Vec<(usize, SeatState, AgentOutcome)> {
14299        let prompts = Prompts::default();
14300        let ctx = WaveCtx {
14301            run: "r",
14302            node: "implement",
14303            prompts: &prompts,
14304            cache,
14305            round: None,
14306            carry_seats: false,
14307            pauses: None,
14308        };
14309        wave(jobs, sem, &ctx, st, 0).await
14310    }
14311
14312    #[tokio::test]
14313    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
14314        crate::run::pin_test_home();
14315        let tmp = tempfile::tempdir().unwrap();
14316        let cache = tmp.path().join("target-a");
14317        let guard = hold(&cache);
14318        // Held for longer than the agent's whole 1s budget.
14319        tokio::spawn(async move {
14320            tokio::time::sleep(Duration::from_millis(1500)).await;
14321            guard.release();
14322        });
14323        let mut st = state(60);
14324        let results = run(
14325            vec![job(
14326                sh("alpha", "echo ok"),
14327                tmp.path(),
14328                Duration::from_secs(1),
14329            )],
14330            Arc::new(Semaphore::new(1)),
14331            Some(&cache),
14332            &mut st,
14333        )
14334        .await;
14335        assert!(
14336            matches!(results[0].2, AgentOutcome::Ok(_)),
14337            "the wait must not be charged to the agent"
14338        );
14339        assert!(st.resource_block.is_none());
14340        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
14341        assert!(st.resource_wait.is_none() && st.active.is_empty());
14342    }
14343
14344    #[tokio::test]
14345    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
14346        crate::run::pin_test_home();
14347        let tmp = tempfile::tempdir().unwrap();
14348        let cache = tmp.path().join("target-b");
14349        let _held = hold(&cache);
14350        let mut st = state(1);
14351        let results = run(
14352            vec![job(
14353                sh("alpha", "echo ok"),
14354                tmp.path(),
14355                Duration::from_secs(1),
14356            )],
14357            Arc::new(Semaphore::new(1)),
14358            Some(&cache),
14359            &mut st,
14360        )
14361        .await;
14362        let (_, seat, out) = &results[0];
14363        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14364        assert!(wave_blocked(&results));
14365        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
14366        assert_eq!(seat.turns, 0, "nothing was asked");
14367        assert!(
14368            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
14369            "no agent was spawned"
14370        );
14371        assert!(FailClass::of(out).is_none(), "not a failure class");
14372        let block = st.resource_block.as_ref().expect("recorded for the resume");
14373        assert_eq!(block.kind, "shared build cache");
14374        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
14375        assert!(st.active.is_empty());
14376    }
14377
14378    #[tokio::test]
14379    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
14380        crate::run::pin_test_home();
14381        let tmp = tempfile::tempdir().unwrap();
14382        let mut st = state(1);
14383        let results = run(
14384            vec![job(
14385                sh("alpha", "echo ok"),
14386                tmp.path(),
14387                Duration::from_secs(1),
14388            )],
14389            Arc::new(Semaphore::new(0)),
14390            None,
14391            &mut st,
14392        )
14393        .await;
14394        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
14395        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
14396        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
14397    }
14398
14399    #[tokio::test]
14400    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
14401        crate::run::pin_test_home();
14402        let tmp = tempfile::tempdir().unwrap();
14403        let mut st = state(60);
14404        let results = run(
14405            vec![job(
14406                sh("alpha", "sleep 30"),
14407                tmp.path(),
14408                Duration::from_secs(1),
14409            )],
14410            Arc::new(Semaphore::new(1)),
14411            None,
14412            &mut st,
14413        )
14414        .await;
14415        let out = &results[0].2;
14416        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
14417        let class = FailClass::of(out).expect("a failure");
14418        assert_eq!(class, FailClass::Timeout);
14419        assert!(should_hand_over(None, &class), "existing handover policy");
14420        assert!(st.resource_block.is_none());
14421    }
14422}