Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// How much verification output is kept and fed back to the fixer.
64const OUTPUT_TAIL: usize = 8_000;
65
66/// Bytes of a failing command's output kept in an event, so the reason a run
67/// stopped is readable from the report without opening `run.json`.
68const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
71/// command's pid before releasing the build cache's lease.
72const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
75/// command's pid to actually exit before giving up and releasing anyway.
76///
77/// A timeout means the process was asked to die (`kill_on_drop`,
78/// `start_kill`), not that it already has — on Windows in particular that can
79/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
80/// the instant the command returns would let the very next acquirer (this
81/// run's own next round, another run's verification, the janitor's prune)
82/// start touching the same directory while it might still be writing to it,
83/// so this polls the actual pid — real confirmation, not a fixed guess —
84/// until it is gone or this ceiling is reached. It is still not full
85/// process-tree reaping: a grandchild the timed-out process spawned and that
86/// outlives it independently is invisible to a pid check, and continuing to
87/// observe and collect *that* stays a different piece of work with its own
88/// owner. Set generously because the common case returns early the moment
89/// the pid is confirmed gone, not because every timeout pays this in full.
90const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92/// Consecutive review rounds with no tree progress (see
93/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
94/// instead of spending the rest of the round budget.
95///
96/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
97/// legitimately finds nothing left to change (its previous round's fix already
98/// covered it, and this round's reviewers re-raised only nits) looks the same
99/// as one that is spinning, for exactly one round. Two in a row is where the
100/// two stop being distinguishable, and a review round on this workload has
101/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
102/// third attempt at a tree that has not moved twice running is pure cost.
103/// This does not touch `review_rounds` itself, which stays the operator's
104/// call.
105pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
108/// a base that moved before giving up and leaving the run `Blocked` for a
109/// person.
110///
111/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
112/// that keeps moving faster than a run can catch it is not something more
113/// rebasing fixes, it is a person's call. Not the same *number as*
114/// `land_rounds` - this budget is spent before a pull request exists, land's
115/// after - but bounded for the identical reason, so it uses the same
116/// default. Counted across both call sites in [`Runner::finish_after_tally`]
117/// (once before review, once before the gate), because either one finding
118/// the base still moving is the same signal.
119const BASE_SYNC_ROUNDS: usize = 4;
120
121/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
122/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
123/// reply held no [`FixReport`].
124///
125/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
126/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
127/// "I'll pause here until the `cargo make check` background run reports
128/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
129/// No `FixReport` was ever collected from that seat, and the run moved on to
130/// the next review round regardless.
131///
132/// Bounded independently of `review_rounds` and `graph.retries`: this
133/// recovers one seat's missing report mid-round, not a new round of review or
134/// an ordinary parse retry, and must not itself become the unbounded wait the
135/// rest of this module exists to avoid.
136const MAX_FIX_CONTINUATIONS: usize = 2;
137
138/// One queued agent invocation.
139///
140/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
141/// CLI hung up on its own stream is asked again from the same job rather than
142/// rebuilt from scratch. See [`Runner::resume_undelivered`].
143#[derive(Clone)]
144struct SeatJob {
145    spec: AgentSpec,
146    seat: SeatState,
147    cwd: PathBuf,
148    prompt: String,
149    timeout: Duration,
150    allow_write: bool,
151    sessions: bool,
152    artifacts: PathBuf,
153    stem: String,
154    /// The prompt for a seat that has been handed to another roster agent
155    /// (a fresh session): everything the original seat would have
156    /// remembered. `None` when `prompt` already carries it, as the first
157    /// ranking and the implement prompt do. Never a resume-style prompt.
158    handover: Option<String>,
159}
160
161/// How the graph reads one agent invocation.
162///
163/// Quota is split out from an ordinary failure on purpose: a rate-limited call
164/// is known to fail again if retried now, so the retry loop must not spend an
165/// attempt on it. `Dropped` is split out for the opposite reason: unlike
166/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
167/// error JSON, never the agent's answer — a caller that matched only
168/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
169/// left to read that JSON as if it were usable output. `resume_undelivered`
170/// is the only caller that acts on it; everywhere else it is reported like an
171/// ordinary failure.
172enum AgentOutcome {
173    /// A usable output.
174    Ok(AgentOutput),
175    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
176    Quota(AgentOutput),
177    /// The CLI hung up on its own stream after billed work. See
178    /// [`agent::AgentOutput::work_undelivered`].
179    Dropped(AgentOutput),
180    /// Any other failure: a timeout, a bad exit code, an empty reply.
181    Failed(String),
182    /// A resource prevented admission or interrupted execution. Not an agent
183    /// failure: never advance a fallback chain or spend an attempt on it.
184    /// Post-spawn interruptions carry a typed error into wave, which records
185    /// implementation preservation before returning this outcome. This outcome
186    /// alone is not proof that no source changes were made.
187    /// [`RunState::resource_block`] says why.
188    ResourceBlocked(String),
189}
190
191/// A request to park the run at its next node boundary.
192///
193/// Cloning is how the request travels: the loop keeps one handle and hands a
194/// clone to each [`Runner`], and every clone points at the same flag. There
195/// is no channel because there is nothing to send - the only message is
196/// "park", it is idempotent, and a flag cannot be missed by a receiver that
197/// was not listening yet.
198///
199/// The boundary is what makes this cheap. Every node writes the run's state
200/// before the next one starts, and every node skips what is already recorded:
201/// `prep` returns early once candidates exist, `implement` asks only the seats
202/// with nothing on disk, `judge` returns early once judgements exist. So a
203/// parked run resumes into exactly the node it stopped before, and no agent
204/// work is thrown away. Killing the process mid-node, by contrast, loses
205/// whatever the seats in flight had not yet written - which for an implement
206/// wave is an hour of paid work.
207///
208/// A [`Runner`] watches two independent handles of this type - see
209/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
210/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
211/// clone covering the whole daemon's lifetime and is never asked to un-park,
212/// which is correct exactly because nothing is dispatched after it fires.
213/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
214/// that parks for an interrupted task must go on to run other tasks
215/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
216/// reusing the daemon-wide one.
217#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221    /// A pause nobody has asked for yet.
222    #[must_use]
223    pub fn new() -> Self {
224        Self::default()
225    }
226
227    /// Ask the run to park at its next node boundary. Idempotent.
228    pub fn park(&self) {
229        self.0.store(true, Ordering::SeqCst);
230    }
231
232    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
233    /// fold into the run's own `park` event - so an operator reading the run
234    /// later knows this was a deliberate interrupt rather than a shutdown or
235    /// a binary swap. The first reason recorded wins; a park already in
236    /// flight is not relabelled by a second, unrelated request.
237    pub fn park_because(&self, reason: impl Into<String>) {
238        self.park_for(ParkCause::Interrupt {
239            reason: reason.into(),
240        });
241    }
242
243    /// [`Pause::park`], recording the structured cause. The first cause
244    /// recorded wins, like [`Pause::park_because`].
245    pub fn park_for(&self, cause: ParkCause) {
246        let mut reason_guard = self
247            .1
248            .lock()
249            .unwrap_or_else(std::sync::PoisonError::into_inner);
250        if reason_guard.is_none() {
251            *reason_guard = Some(cause);
252        }
253        drop(reason_guard);
254        self.park();
255    }
256
257    /// Has a park been asked for?
258    #[must_use]
259    pub fn parked(&self) -> bool {
260        self.0.load(Ordering::SeqCst)
261    }
262
263    /// Why the park was asked for, when the caller used [`Pause::park_because`].
264    #[must_use]
265    pub fn reason(&self) -> Option<String> {
266        self.cause().map(|c| c.label())
267    }
268
269    /// The structured cause, when one was recorded.
270    #[must_use]
271    pub fn cause(&self) -> Option<ParkCause> {
272        self.1
273            .lock()
274            .unwrap_or_else(std::sync::PoisonError::into_inner)
275            .clone()
276    }
277}
278
279/// Drives one run.
280pub struct Runner {
281    /// Run state; public so the CLI can report on it.
282    pub state: RunState,
283    roles: ResolvedRoles,
284    sem: Arc<Semaphore>,
285    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
286    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
287    /// this is never the same handle as `interrupt`.
288    pause: Pause,
289    /// Set when `magi serve`'s interrupt scheduler wants this specific run
290    /// parked at its next node boundary, to let a task marked
291    /// [`crate::queue::Task::interrupt`] run alone before this one carries
292    /// on. Unlike `pause`, a fresh, unshared handle per run - see
293    /// [`Runner::watch_interrupt`].
294    interrupt: Pause,
295}
296
297/// Where the branch's own commits start: its merge base with the base branch
298/// as the remote has it now, else with the recorded `base_commit`. A branch
299/// rebased onto a base that moved past `base_commit` would otherwise count
300/// the base's commits as its own under `base_commit..branch`.
301async fn review_base(
302    repo: &Path,
303    remote: &str,
304    base_branch: &str,
305    base_commit: &str,
306    branch: &str,
307) -> String {
308    review_base_checked(repo, remote, base_branch, base_commit, branch)
309        .await
310        .0
311}
312
313/// [`review_base`] plus whether the base was read from a freshly fetched
314/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
315/// never older than `base_commit`'s view of the base), but the answer is then
316/// not trusted to rewrite a pull request's title.
317async fn review_base_checked(
318    repo: &Path,
319    remote: &str,
320    base_branch: &str,
321    base_commit: &str,
322    branch: &str,
323) -> (String, bool) {
324    let tracking = format!("{remote}/{base_branch}");
325    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326    if git::rev_exists(repo, &tracking).await
327        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328        && !mb.is_empty()
329    {
330        return (mb, fresh);
331    }
332    let mb = git::merge_base(repo, base_commit, branch)
333        .await
334        .ok()
335        .filter(|mb| !mb.is_empty())
336        .unwrap_or_else(|| base_commit.to_owned());
337    (mb, false)
338}
339
340/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
341/// when git cannot say or finds nothing: a stale list is better than a wrong
342/// or empty one.
343pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344    if !is_review_run(state) {
345        return;
346    }
347    let base = review_base(
348        &state.repo,
349        &state.config.merge.remote,
350        &state.base_branch,
351        &state.base_commit,
352        branch,
353    )
354    .await;
355    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356        && !subjects.is_empty()
357        && state.reviewed_commits.as_ref() != Some(&subjects)
358    {
359        state.reviewed_commits = Some(subjects);
360        state.save().ok();
361    }
362}
363
364/// Subjects of the base's commits between the recorded start and the branch's
365/// merge base: what a stale `base_commit..branch` would have mistaken for the
366/// branch's own work.
367async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368    let (base, trusted) = review_base_checked(
369        &state.repo,
370        &state.config.merge.remote,
371        &state.base_branch,
372        &state.base_commit,
373        branch,
374    )
375    .await;
376    if !trusted {
377        return None;
378    }
379    git::subjects(&state.repo, &state.base_commit, &base)
380        .await
381        .ok()
382}
383
384/// May an adopted pull request's title be replaced with `computed`? Only when
385/// it is empty, magi's own shape, or a base commit's subject that leaked in;
386/// a title a person wrote stays. Never when `computed` is itself a leak.
387pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389    if is_leak(computed) {
390        return false;
391    }
392    let cur = current.trim();
393    cur.is_empty()
394        || cur.starts_with(REVIEW_PROMPT_OPENING)
395        || cur.starts_with("chore: land candidate")
396        || cur.starts_with("magi: candidate")
397        || is_leak(cur)
398}
399
400/// The commit a run branches from: the base branch as the remote has it.
401///
402/// Two failures this replaces. A run used to branch off `HEAD` and so refused
403/// to start on a dirty tree, which made `magi serve` decline every task for as
404/// long as the operator had work in progress - most of the time. Branching off
405/// the *local* base branch fixed that and introduced a worse one: `land` merges
406/// the winner on GitHub, nothing updates the local ref, and the next run
407/// branches off a base missing everything the previous runs landed. Two tasks
408/// in a row from a phone would have had the second silently re-implementing
409/// against stale code and opening a pull request that reverted the first.
410///
411/// Only refs move here - no checkout, no local branch, no merge - so it is safe
412/// with uncommitted work in the tree. A machine with no network still starts:
413/// the fetch may fail and the local tip is used with a warning, because
414/// refusing to run offline is a worse failure than running against a base the
415/// operator can see for themselves.
416///
417/// One function, called by both entry points. Two answers to "where does a run
418/// branch from" is the kind of drift nobody notices until a diff is wrong.
419/// Bring the local `branch` in line with `<remote>/<branch>` before a review
420/// checks it out.
421///
422/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
423/// anything other than plain `git push` from this checkout (a jj colocated
424/// workspace, another clone) moves only the remote-tracking ref - so the local
425/// one can be a stale placeholder. It moves only when local is behind the remote or is an
426/// empty placeholder that diverged from it; unpushed local work is kept, and a real
427/// divergence is refused rather than guessed at.
428async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429    let tracking = format!("{remote}/{branch}");
430    let fetched = git::fetch(repo, remote, branch).await;
431    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432    let local_exists = git::branch_exists(repo, branch).await?;
433    if !fresh {
434        if !local_exists {
435            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436        }
437        tracing::warn!(
438            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439        );
440        return Ok(());
441    }
442    let remote_sha = git::rev_parse(repo, &tracking).await?;
443    if !local_exists {
444        git::git(repo, &["branch", branch, &tracking]).await?;
445        return Ok(());
446    }
447    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449        return Ok(());
450    }
451    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452        // Diverged. `reconcile` settles it only when it can prove nothing is
453        // lost: a local tip that is the remote's change rebased is pushed over
454        // it (lease pinned to the tip read here), a tip whose every extra
455        // commit is empty is a placeholder the remote's work replaced, and
456        // anything else is two different changes - a question for a person.
457        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458            .await?
459        {
460            crate::reconcile::Reconciliation::Pushed => {
461                tracing::warn!(
462                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463                    short(&local_sha),
464                    short(&remote_sha)
465                );
466                return Ok(());
467            }
468            crate::reconcile::Reconciliation::Placeholder => {}
469            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470        }
471    }
472    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473    if !out.ok() {
474        bail!(
475            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476            short(&local_sha),
477            short(&remote_sha),
478            out.stderr
479        );
480    }
481    tracing::warn!(
482        "local `{branch}` was stale: fast-forwarded {} -> {}",
483        short(&local_sha),
484        short(&remote_sha)
485    );
486    Ok(())
487}
488
489async fn resolve_base(
490    repo: &Path,
491    base_branch: &str,
492    remote: &str,
493    coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495    let _lease = crate::repo_lock::SeatLease::operation_with(
496        repo,
497        coordination,
498        &crate::repo_lock::WaitPolicy::current(),
499    )
500    .await?;
501    let tracking = format!("{remote}/{base_branch}");
502    let fetched = git::fetch(repo, remote, base_branch).await;
503    if let Ok(out) = &fetched
504        && out.ok()
505        && git::rev_exists(repo, &tracking).await
506    {
507        return Ok(crate::run::FetchedBase {
508            remote: remote.to_owned(),
509            branch: base_branch.to_owned(),
510            commit: git::rev_parse(repo, &tracking).await?,
511            fetched_at: jiff::Timestamp::now(),
512        });
513    }
514    let why = match &fetched {
515        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516        Ok(_) => format!("{remote} has no {base_branch}"),
517        Err(e) => e.to_string(),
518    };
519    // No fallback to the local branch: it may be behind, and branching off an
520    // old commit is the stale-checkout bug this check exists to prevent.
521    bail!(
522        "cannot read {tracking} ({why}); refusing to branch off the local \
523         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524         base / remote in magi.toml. This is launcher preparation: check the \
525         launcher's Git administration access, network and repository coordination \
526         before retrying; an unattended seat cannot approve those permissions"
527    )
528}
529
530/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
531/// on an early return or a panic.
532///
533/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
534/// manual `magi fix` invocations against the same run are otherwise
535/// invisible to each other and would race to remove and recreate the same
536/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
537/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
538/// which is only ever reclaimed later, out of band, by
539/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
540/// `magi fix` invocation is not necessarily running under either of those, so
541/// nothing would ever sweep a lock a killed or crashed process left behind.
542/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
543/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
544/// lease use: an unreadable or unparsable pid, or a liveness query the
545/// platform cannot answer, reads as alive and the lock is left in place.
546struct FixClaim {
547    path: PathBuf,
548}
549
550impl FixClaim {
551    fn acquire(dir: &Path) -> Result<Self> {
552        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
553        let path = dir.join("fix.lock");
554        let retry = crate::lock_file::Retry::new(None);
555        match Self::create(&path, retry) {
556            Ok(claim) => Ok(claim),
557            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
558                if Self::reclaim_if_dead(&path) {
559                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
560                } else {
561                    bail!(
562                        "another repair or `magi fix` is already running for this run ({} exists)",
563                        path.display()
564                    )
565                }
566            }
567            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
568        }
569    }
570
571    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
572        let mut f = retry.create(path)?;
573        use std::io::Write as _;
574        // Read back by `reclaim_if_dead` on a later, stuck invocation.
575        writeln!(f, "{}", std::process::id())?;
576        Ok(Self {
577            path: path.to_owned(),
578        })
579    }
580
581    /// True if the lock named a process confirmed dead, in which case it was
582    /// also removed. Never true on an unreadable file, an unparsable pid, or
583    /// a liveness query the platform cannot answer — see this type's own doc.
584    fn reclaim_if_dead(path: &Path) -> bool {
585        let dead = std::fs::read_to_string(path)
586            .ok()
587            .and_then(|body| body.trim().parse::<u32>().ok())
588            .is_some_and(|pid| !crate::proc::pid_alive(pid));
589        dead && std::fs::remove_file(path).is_ok()
590    }
591}
592
593impl Drop for FixClaim {
594    fn drop(&mut self) {
595        let _ = std::fs::remove_file(&self.path);
596    }
597}
598
599/// Holds the mutable runner as well as its durable lease: cancellation must
600/// clear both copies of ownership, including state a caller may save later.
601struct DriverExecution<'a> {
602    runner: &'a mut Runner,
603    driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607    fn drop(&mut self) {
608        self.driver.release_local(&mut self.runner.state);
609    }
610}
611
612impl Runner {
613    /// Start a fresh run against `repo`.
614    pub async fn start(
615        repo: &Path,
616        instruction: String,
617        config: Config,
618        origin: Origin,
619    ) -> Result<Self> {
620        Self::start_naming(repo, instruction, "", config, origin).await
621    }
622
623    /// [`Runner::start`] for a queued task. Titles and instructions never
624    /// select implementation sources. The title argument is retained for callers.
625    pub async fn start_naming(
626        repo: &Path,
627        instruction: String,
628        _also_scan: &str,
629        config: Config,
630        origin: Origin,
631    ) -> Result<Self> {
632        let repo = git::toplevel(repo).await?;
633        let missing = agent::missing_programs(&config.agents);
634        if !missing.is_empty() {
635            bail!(
636                "these agent programs are not on PATH: {}. Fix the roster in \
637                 magi.toml or install them.",
638                missing.join(", ")
639            );
640        }
641        let base_branch =
642            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
643                .await?;
644        let fetched_base = resolve_base(
645            &repo,
646            &base_branch,
647            &config.merge.remote,
648            config.coordination.as_ref(),
649        )
650        .await?;
651        let base_commit = fetched_base.commit.clone();
652        let roles = config.resolve_roles()?;
653        let max_parallel = config.graph.max_parallel.max(1);
654        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
655        // Recorded before the first save, so a crash right after minting
656        // cannot leave a run with no origin. Legibility only: nothing reads it
657        // to decide anything.
658        state.fetched_base = Some(fetched_base);
659        state.origin = Some(origin);
660        state.event("start", format!("run {} created", state.id));
661        state.save()?;
662        Ok(Self {
663            state,
664            roles,
665            sem: Arc::new(Semaphore::new(max_parallel)),
666            pause: Pause::new(),
667            interrupt: Pause::new(),
668        })
669    }
670
671    /// Open a review-only run against work that already exists on `branch`.
672    ///
673    /// The expensive half of the graph is the implement wave — measured at
674    /// 111 and 134 internal tool-loop turns on this repository, against a
675    /// handful for a judge or a reviewer. The cheap half is worth running on
676    /// hand-written work too, and there was no way to reach it.
677    ///
678    /// No new state and no schema change are needed: a run with **one** viable
679    /// candidate and a tally already decided degrades `execute` to exactly
680    /// review → gate → merge, because `judge` skips a single-candidate field,
681    /// `deliberate` has fewer than two first choices to reconcile, `vote`
682    /// returns early, `tally` is already present and `fold_losers` has no
683    /// losers. Resuming such a run therefore does the right thing as well.
684    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
685        Self::review_taking_over(repo, branch, config, None, origin).await
686    }
687
688    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
689    /// at the same task still has `branch` checked out, its worktree is
690    /// released first if that is safe (see [`crate::handover`]), and the
691    /// review refuses with the reason if it is not. `None` is a hand-run
692    /// review: it has no earlier attempts, so only a worktree of a dead run
693    /// magi recorded itself can be released.
694    pub async fn review_taking_over(
695        repo: &Path,
696        branch: &str,
697        config: Config,
698        takeover: Option<crate::handover::Takeover>,
699        origin: Origin,
700    ) -> Result<Self> {
701        let repo = git::toplevel(repo).await?;
702        let missing = agent::missing_programs(&config.agents);
703        if !missing.is_empty() {
704            bail!(
705                "these agent programs are not on PATH: {}. Fix the roster in \
706                 magi.toml or install them.",
707                missing.join(", ")
708            );
709        }
710        let base_branch =
711            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
712                .await?;
713        if base_branch == branch {
714            bail!("`{branch}` is the base branch; there is nothing to review against");
715        }
716        let fetched_base = resolve_base(
717            &repo,
718            &base_branch,
719            &config.merge.remote,
720            config.coordination.as_ref(),
721        )
722        .await?;
723        let base_commit = fetched_base.commit.clone();
724
725        let roles = config.resolve_roles()?;
726        let max_parallel = config.graph.max_parallel.max(1);
727        let mut state = RunState::new(
728            repo.clone(),
729            base_branch,
730            base_commit.clone(),
731            String::new(),
732            config,
733        );
734        state.fetched_base = Some(fetched_base);
735        state.origin = Some(origin);
736
737        // Released before anything else touches the branch: a stale local
738        // branch is moved with `git branch -f`, which git refuses while an
739        // earlier attempt's worktree still has it checked out. Everything
740        // after this point that can fail puts the old run back.
741        // A hand-run review has no task, hence no earlier attempts, but a
742        // worktree of a dead run magi made may still be released.
743        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
744            earlier: Vec::new(),
745            home: crate::run::home(),
746            choice: None,
747            task_instruction: None,
748        });
749        let wait = repository_wait(&state, None);
750        let coordination_config = state.config.coordination.clone();
751        let coordination =
752            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
753                .await?;
754        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
755        if let Some(released) = &released {
756            state.event(
757                "release",
758                format!(
759                    "took `{branch}` over from run {}: its worktree was released: {}",
760                    crate::run::short_of(&released.old_id),
761                    released.audit
762                ),
763            );
764        }
765        // The owner's answer to an earlier divergence question is applied
766        // here: after the release (git will not move a checked-out branch)
767        // and before the sync that would otherwise ask again.
768        if let Some(choice) = takeover.choice.as_ref()
769            && let Err(e) =
770                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
771                    .await
772        {
773            if let Some(released) = &released {
774                released.restore(&repo, branch).await;
775            }
776            return Err(e.context("applying the owner's answer about the diverged branch"));
777        }
778        // An earlier attempt of this task that published the task head, and
779        // whose work is the branch under review, hands that head over: the
780        // review run updates the same pull request. Without one (a legacy
781        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
782        // own head and its own open pull request is adopted as before.
783        for id in takeover.earlier.iter().rev() {
784            if let Ok(old) = RunState::load_under(id, &takeover.home)
785                && let Some(head) = old.pr_head.clone()
786                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
787            {
788                state.pr_head = Some(head);
789                break;
790            }
791        }
792        drop(coordination);
793        let opened = Self::open_review(
794            &repo,
795            branch,
796            state,
797            roles,
798            max_parallel,
799            base_commit,
800            &takeover,
801        )
802        .await;
803        if opened.is_err()
804            && let Some(released) = &released
805        {
806            let _coordination = crate::repo_lock::SeatLease::operation_with(
807                &repo,
808                coordination_config.as_ref(),
809                &wait,
810            )
811            .await?;
812            released.restore(&repo, branch).await;
813        }
814        opened
815    }
816
817    /// Recover exactly the operator-selected public implementation, without touching old sources.
818    pub async fn recover_pr(
819        repo: &Path,
820        task: &str,
821        selection: crate::pr_recovery::Selection,
822        mut config: Config,
823    ) -> Result<Self> {
824        selection.configure(&mut config)?;
825        selection.check_task(task, &selection.instruction)?;
826        let base_branch =
827            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
828                .await?;
829        let fetched_base = resolve_base(
830            repo,
831            &base_branch,
832            &config.merge.remote,
833            config.coordination.as_ref(),
834        )
835        .await?;
836        let base_commit = fetched_base.commit.clone();
837        selection
838            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
839            .await?;
840        Box::pin(selection.verify_source(repo)).await?;
841        if let Some(proof) = &selection.source_proof {
842            anyhow::ensure!(
843                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
844                "source rebase base is outside the fetched landing base"
845            );
846        }
847        let roles = config.resolve_roles()?;
848        let max_parallel = config.graph.max_parallel.max(1);
849        let mut state = RunState::new(
850            repo.to_path_buf(),
851            base_branch,
852            base_commit.clone(),
853            selection.instruction.clone(),
854            config,
855        );
856        state.fetched_base = Some(fetched_base);
857        state.origin = Some(crate::run::Origin::queue(task));
858        state.pr_head = Some(selection.branch.clone());
859        state.pr_recovery = Some(selection.clone());
860        crate::pr_head::check_predecessors(&state, &selection.branch).await?;
861        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
862    }
863
864    async fn open_selected(
865        repo: &Path,
866        state: RunState,
867        roles: ResolvedRoles,
868        max_parallel: usize,
869        base_commit: String,
870        selection: crate::pr_recovery::Selection,
871    ) -> Result<Self> {
872        let mut state = state;
873        let branch = state.branch_for('A');
874        let wait = repository_wait(&state, None);
875        let coordination = coordinate_operation(&mut state, repo, &wait).await?;
876        git::git(repo, &["branch", &branch, selection.source()]).await?;
877        drop(coordination);
878        state.event(
879            "recovery",
880            format!(
881                "selected implementation {} for {} (public lease {})",
882                selection.source(),
883                selection.url,
884                selection.head
885            ),
886        );
887        let takeover = crate::handover::Takeover {
888            earlier: Vec::new(),
889            home: crate::run::home(),
890            choice: None,
891            task_instruction: Some(selection.instruction.clone()),
892        };
893        state.save()?;
894        let id = state.id.clone();
895        let opened = Self::open_review(
896            repo,
897            &branch,
898            state,
899            roles,
900            max_parallel,
901            base_commit,
902            &takeover,
903        )
904        .await;
905        match opened {
906            Ok(runner) => Ok(runner),
907            Err(e) => {
908                if let Ok(mut partial) = RunState::load(&id) {
909                    partial.status = RunStatus::Blocked;
910                    partial.driver_exited = true;
911                    partial.event(
912                        "recovery",
913                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
914                    );
915                    partial.save()?;
916                }
917                Err(crate::pr_recovery::StartupFailed {
918                    run: id,
919                    cause: format!("{e:#}"),
920                }
921                .into())
922            }
923        }
924    }
925
926    /// What the newest earlier attempt that reviewed `branch` leaves for this
927    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
928    /// no earlier attempts. Reads the earlier record only: ownership of its
929    /// worktree and branch is [`crate::handover`]'s business.
930    async fn inherit(
931        repo: &Path,
932        branch: &str,
933        base_commit: &str,
934        takeover: &crate::handover::Takeover,
935    ) -> Option<crate::recovery::RecoveryRecord> {
936        use crate::recovery::{Relation, assess};
937        let newest = takeover.earlier.last()?;
938        let mut chosen = None;
939        for id in takeover.earlier.iter().rev() {
940            if let Ok(old) = RunState::load_under(id, &takeover.home)
941                && !old.reviews.is_empty()
942                && (old.winner().is_some_and(|w| w.branch == branch)
943                    || old.pr_head.as_deref() == Some(branch))
944            {
945                chosen = Some(old);
946                break;
947            }
948        }
949        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
950            .await
951            .unwrap_or_default();
952        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
953            Some(last) if last.head == tip => Relation::Same,
954            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
955            _ => Relation::Diverged,
956        };
957        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
958        Some(assess(from, chosen.as_ref(), base_commit, relation))
959    }
960
961    /// The half of [`Runner::review_taking_over`] that can fail after an
962    /// earlier attempt's worktree was released.
963    async fn open_review(
964        repo: &Path,
965        branch: &str,
966        mut state: RunState,
967        roles: ResolvedRoles,
968        max_parallel: usize,
969        base_commit: String,
970        takeover: &crate::handover::Takeover,
971    ) -> Result<Self> {
972        let wait = repository_wait(&state, None);
973        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
974        if state.pr_recovery.is_none() {
975            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
976        }
977        // The commit subjects are the closest thing to a task statement that
978        // existing work carries, and the reviewers are told as much.
979        let start = review_base(
980            repo,
981            &state.config.merge.remote,
982            &state.base_branch,
983            &base_commit,
984            branch,
985        )
986        .await;
987        let log = git::log_oneline(repo, &start, branch)
988            .await
989            .unwrap_or_default();
990        let log_text = if log.trim().is_empty() {
991            "(no commit messages)"
992        } else {
993            log.trim()
994        };
995        let mut instruction = format!(
996            "Review the work already on branch `{branch}`. There is no task \
997             statement: what the change claims to do is whatever its commits \
998             say.\n\n{log_text}"
999        );
1000        // A retry of a task whose earlier attempt left reviews behind carries
1001        // that attempt's evidence, and the task's own statement with it: the
1002        // completion criteria are the task's, not whatever the commits claim.
1003        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1004            if let Some(task) = takeover
1005                .task_instruction
1006                .as_deref()
1007                .filter(|t| !t.trim().is_empty())
1008                && rec.reviewed_head.is_some()
1009            {
1010                instruction = format!(
1011                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1012                    task.trim()
1013                );
1014                rec.criteria_inherited = true;
1015            }
1016            state.event("recovery", rec.summary());
1017            state.recovery = Some(rec);
1018        }
1019        if let Some(selected) = &state.pr_recovery {
1020            instruction = selected.instruction.clone();
1021        }
1022        state.task_instruction = takeover.task_instruction.clone();
1023        state.instruction = instruction;
1024        state.reviewed_commits = Some(
1025            git::subjects(repo, &start, branch)
1026                .await
1027                .unwrap_or_default(),
1028        );
1029
1030        // A takeover must not forget a hold that a previous review observed,
1031        // even if repairs have since removed it from the branch.
1032        for id in &takeover.earlier {
1033            if let Ok(old) = RunState::load_under(id, &takeover.home)
1034                && old.repo == state.repo
1035                && (old.pr_head.as_deref() == Some(branch)
1036                    || old.winner().is_some_and(|w| w.branch == branch))
1037            {
1038                for hold in old.branch_owner_holds {
1039                    if !state.branch_owner_holds.contains(&hold) {
1040                        state.event(
1041                            "approval-policy",
1042                            format!(
1043                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1044                                hold.control.key(),
1045                                hold.file,
1046                                hold.head
1047                            ),
1048                        );
1049                        state.branch_owner_holds.push(hold);
1050                    }
1051                }
1052            }
1053        }
1054        let reviewed_head = git::rev_parse(repo, branch).await?;
1055        state.observe_branch_owner_holds(&reviewed_head).await;
1056
1057        // An attached worktree, so the fixer's commits land on the branch under
1058        // review rather than on a detached head nobody will look at again.
1059        let worktree = state.worktree_root().join("under-review");
1060        if let Some(parent) = worktree.parent() {
1061            tokio::fs::create_dir_all(parent).await.ok();
1062        }
1063        let path = worktree.to_string_lossy().to_string();
1064        git::git(repo, &["worktree", "add", &path, branch])
1065            .await
1066            .with_context(|| {
1067                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1068            })?;
1069
1070        let setup = match ensure_setup_config(&mut state, repo).await {
1071            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1072            Err(e) => Err(e),
1073        };
1074        if let Err(e) = setup {
1075            git::worktree_remove_clean(repo, &worktree).await.ok();
1076            return Err(e);
1077        }
1078        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1079            .await
1080            .unwrap_or(0);
1081        if commits == 0 {
1082            git::worktree_remove_clean(repo, &worktree).await.ok();
1083            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1084        }
1085        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1086            .await
1087            .map(|f| f.len())
1088            .unwrap_or(0);
1089        if files == 0
1090            && let (Ok(head_tree), Ok(base_tree)) = (
1091                git::tree_of(&worktree, "HEAD").await,
1092                git::tree_of(&worktree, &base_commit).await,
1093            )
1094            && head_tree == base_tree
1095        {
1096            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1097            git::worktree_remove_clean(repo, &worktree).await.ok();
1098            bail!(
1099                "`{branch}` at {} has a tree identical to base {}; this usually means \
1100                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1101                 against `{}/{branch}`) rather than an empty change",
1102                short(&head),
1103                short(&base_commit),
1104                state.config.merge.remote
1105            );
1106        }
1107        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1108            .await
1109            .unwrap_or_default();
1110
1111        state.candidates.push(Candidate {
1112            index: 0,
1113            label: 'A',
1114            // Not an agent id on purpose: nothing in the roster wrote this, and
1115            // the stats tables must not credit anyone with a win for it.
1116            agent: EXISTING_BRANCH.to_owned(),
1117            branch: branch.to_owned(),
1118            worktree,
1119            summary: String::new(),
1120            stat,
1121            files,
1122            commits,
1123            empty: false,
1124            failed: None,
1125            verified_noop: None,
1126            duration_ms: 0,
1127            folded: false,
1128        });
1129        state.tally = Some(Tally {
1130            first_choice: BTreeMap::from([('A', 0)]),
1131            borda: BTreeMap::new(),
1132            winner: 'A',
1133            rankings: 0,
1134            unanimous_initial: false,
1135            deliberated: false,
1136            changed_votes: 0,
1137            unanimous_final: false,
1138            tie_break: None,
1139            // No panel sat, so no quorum applies. Zero judges is the correct
1140            // number for work that never competed, and must not be reported as
1141            // a collapsed panel.
1142            judges: 0,
1143            present: 0,
1144            quorum: 0,
1145            met_quorum: true,
1146            uncontested: Some("review-only run: nothing competed".to_owned()),
1147        });
1148        state.status = RunStatus::Reviewing;
1149        state.prep_complete = Some(true);
1150        state.event(
1151            "start",
1152            format!(
1153                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1154                state.id
1155            ),
1156        );
1157        state.save()?;
1158        Ok(Self {
1159            state,
1160            roles,
1161            sem: Arc::new(Semaphore::new(max_parallel)),
1162            pause: Pause::new(),
1163            interrupt: Pause::new(),
1164        })
1165    }
1166
1167    /// Reopen an existing run.
1168    pub fn resume(id: &str) -> Result<Self> {
1169        let state = RunState::load(id)?;
1170        crate::pr_head::ensure_owner(&state)?;
1171        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1172            bail!(
1173                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1174            );
1175        }
1176        if let Some(to) = &state.released_to {
1177            bail!(
1178                "run {} cannot be resumed: its worktree was released to run {}",
1179                state.short(),
1180                crate::run::short_of(to)
1181            );
1182        }
1183        let roles = state.config.resolve_roles()?;
1184        if let Some(id) = &state.config.graph.selected_implementer
1185            && state.candidates.iter().any(|c| c.agent != *id)
1186        {
1187            bail!(
1188                "replacement implementer pin does not match the saved candidates; task must stay held"
1189            );
1190        }
1191        let max_parallel = state.config.graph.max_parallel.max(1);
1192        Ok(Self {
1193            state,
1194            roles,
1195            sem: Arc::new(Semaphore::new(max_parallel)),
1196            pause: Pause::new(),
1197            interrupt: Pause::new(),
1198        })
1199    }
1200
1201    /// Walk the graph to a terminal state, skipping nodes already recorded.
1202    ///
1203    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1204    /// the phone - ends here, so this is the one place a run that ended
1205    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1206    /// notification centre. Best-effort: see [`crate::notices::raise`].
1207    pub async fn execute(&mut self) -> Result<()> {
1208        if let Some(selection) = &self.state.pr_recovery
1209            && (selection.source_proof.is_some() || selection.source_run.is_some())
1210        {
1211            Box::pin(selection.verify_source(&self.state.repo)).await?;
1212        }
1213        // Queue claims cover daemon work; this also serializes manual drivers
1214        // consuming a repair answer, including the new review and decision.
1215        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1216            || self.state.land_approval.is_some()
1217            || self.state.merge_decision.is_some()
1218            || !self.state.merge_repairs.is_empty()
1219            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1220        {
1221            let claim = FixClaim::acquire(&self.state.dir())?;
1222            let disk = RunState::load(&self.state.id)?;
1223            let checkpoint = |s: &RunState| {
1224                (
1225                    s.merge_repairs.len(),
1226                    s.merge_repairs
1227                        .last()
1228                        .map(|r| (r.question.clone(), r.phase)),
1229                    s.land_approval
1230                        .as_ref()
1231                        .map(|a| (a.question.clone(), a.head.clone())),
1232                )
1233            };
1234            if checkpoint(&disk) != checkpoint(&self.state) {
1235                bail!(
1236                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1237                );
1238            }
1239            Some(claim)
1240        } else {
1241            None
1242        };
1243        let mut execution = self.claim_driver().await?;
1244        let runner = &mut *execution.runner;
1245        let wait = runner.repository_wait();
1246        // Keep the graph's large future off the callers' stack, including
1247        // standalone task attempts running on a default-sized test thread.
1248        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1249        let result = match result {
1250            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1251                record_repository_block(&mut runner.state, &e);
1252                runner.park_here()?;
1253                Ok(())
1254            }
1255            other => other,
1256        };
1257        execution.driver.finish(&mut runner.state).await?;
1258        let ended = if result.is_err() {
1259            Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1260        } else {
1261            crate::notices::run_ended(&runner.state)
1262        };
1263        if let Some(notice) = ended {
1264            crate::notices::raise(notice);
1265        }
1266        result
1267    }
1268
1269    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1270        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1271    }
1272
1273    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1274        let wait = self.repository_wait();
1275        coordinate_operation(&mut self.state, cwd, &wait).await
1276    }
1277
1278    /// Validate ownership and persist the driver claim in one critical section.
1279    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1280        let owner = match crate::pr_head::owner_lock_wait(
1281            &crate::run::home(),
1282            &self.state.id,
1283            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1284        )
1285        .await
1286        {
1287            Ok(owner) => owner,
1288            Err(error) => {
1289                let detail = format!("worktree maintenance or ownership update: {error:#}");
1290                // Admission did not own run.json: retain this resource outcome
1291                // in memory for queue settlement without overwriting a concurrent
1292                // retirement or publication transaction.
1293                self.state.resource_block = Some(crate::run::ResourceBlock {
1294                    node: "start".into(),
1295                    kind: "worktree maintenance".into(),
1296                    detail: detail.clone(),
1297                    at: Timestamp::now(),
1298                });
1299                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1300            }
1301        };
1302        let disk = RunState::load(&self.state.id)?;
1303        crate::pr_head::ensure_owner(&disk)?;
1304        if disk.liveness(false) == crate::run::Liveness::Live {
1305            bail!("run already has a live driver; wait for its execution to end");
1306        }
1307        if let Some(to) = &disk.released_to {
1308            bail!(
1309                "run {} cannot continue: its worktree was released to run {}",
1310                self.state.short(),
1311                crate::run::short_of(to)
1312            );
1313        }
1314        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1315        let mut execution = DriverExecution {
1316            runner: self,
1317            driver,
1318        };
1319        // Declare after the execution guard: on any claim error this short
1320        // lock must drop before that guard reloads and releases ownership.
1321        let _owner = owner;
1322        let state = &mut execution.runner.state;
1323        // Moving again, so it is no longer parked. Set before the walk rather
1324        // than in `resume`, so every way of re-entering the graph clears it
1325        // and a card cannot claim a run is waiting to be resumed while the
1326        // agents are already working.
1327        state.parked = false;
1328        state.resource_wait = None;
1329        state.resource_block = None;
1330        // Any seat this state still lists as answering belongs to whatever
1331        // process last drove this run — this one included, if it crashed
1332        // mid-wave. Cleared and flushed immediately, before anything else
1333        // runs, so a resume can never show a seat as live when nothing is
1334        // asking it anything yet; the node that actually dispatches the next
1335        // wave repopulates it.
1336        state.clear_active();
1337        // Recorded in the same spot, and flushed together with the clear
1338        // above: this is the pid a reader checks (`RunState::liveness`) when
1339        // no daemon claim exists to answer "is a process still driving this
1340        // run" — a plain `magi run` / `magi review` typed into a terminal
1341        // claims nothing there. Always overwritten, never only-if-absent, so
1342        // a resumed run's stale pid from a previous, possibly-dead process
1343        // can never survive into this one's own report. Unlike
1344        // `clear_active`, this changes on every single `execute()` call, so
1345        // the save below is now unconditional rather than only-if-cleared.
1346        //
1347        // `driver_started_at` is recorded in the same breath, from this same
1348        // pid, so `liveness` can tell a live pid that is genuinely still us
1349        // apart from one the OS has since handed to an unrelated process —
1350        // see that field's own doc for why the pid alone is not enough.
1351        let pid = std::process::id();
1352        state.driver_pid = Some(pid);
1353        state.driver_started_at = crate::proc::process_started_at(pid);
1354        state.driver_exited = false;
1355        state.begin_pass();
1356        execution.driver.bind(state);
1357        state.save()?;
1358        execution.driver.record(state)?;
1359        Ok(execution)
1360    }
1361
1362    async fn execute_graph(&mut self) -> Result<()> {
1363        self.state.configure_recovery_resume()?;
1364        // Preservation may invalidate an old empty classification. Resolve it
1365        // before any resume shortcut can reuse a verdict or publish a branch.
1366        self.persist_candidates().await?;
1367        if self
1368            .state
1369            .merge_decision
1370            .as_ref()
1371            .is_some_and(|decision| !decision.closed)
1372            && self.state.status != RunStatus::Landing
1373        {
1374            if crate::merge_decision::resumable_hold(&self.state) {
1375                self.state.merge = Some(MergeOutcome {
1376                    mode: MergeMode::Pr,
1377                    ok: false,
1378                    detail: self
1379                        .state
1380                        .merge_decision
1381                        .as_ref()
1382                        .unwrap()
1383                        .target
1384                        .pr
1385                        .clone(),
1386                    empty: false,
1387                });
1388                self.run_land().await?;
1389                self.settle_questions();
1390                return Ok(());
1391            }
1392            bail!(
1393                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1394            );
1395        }
1396        if self.state.config.graph.selected_implementer.is_some()
1397            && self.state.status == RunStatus::Blocked
1398            && self.state.tally.is_none()
1399            && self.state.candidate_preservation.is_empty()
1400            && self.state.interrupted_implementations.is_empty()
1401        {
1402            bail!(
1403                "selected implementation is held; explicitly release the owning task to retry it"
1404            );
1405        }
1406
1407        if self.state.merge_repairs.last().is_some_and(|r| {
1408            matches!(
1409                r.phase,
1410                crate::run::MergeRepairPhase::Requested
1411                    | crate::run::MergeRepairPhase::Fixing
1412                    | crate::run::MergeRepairPhase::Reviewing
1413            )
1414        }) {
1415            if self.repair_merge_review().await? {
1416                self.run_land().await?;
1417            }
1418            self.settle_questions();
1419            return Ok(());
1420        }
1421        // Publication already passed review and gate. Resume only that node:
1422        // syncing to a newer base here could change the verified commit.
1423        if self.state.pr_publication_pending {
1424            self.merge().await?;
1425            if self.resource_halt()? {
1426                return Ok(());
1427            }
1428            self.state.save()?;
1429            self.settle_questions();
1430            return Ok(());
1431        }
1432        // A run that already lost its quorum never resumes into the verdict
1433        // machinery: `deliberate` and `vote` would otherwise clobber the
1434        // stalled marker back to Voting and the run would keep going past a
1435        // verdict that is no longer trustworthy. Everything already recorded is
1436        // kept, so the run stays resumable (or foldable) for a human to pick up.
1437        //
1438        // On --resume the run gets one chance to repair itself: the seats a
1439        // rate limit took out are re-asked. If their quota has since reset and
1440        // the quorum is restored, the run picks up and finishes; otherwise it
1441        // stays stale and still-resumable for a later retry. If it does not
1442        // recover, the returned status stays `Stalled` and nothing was
1443        // clobbered (the recovery only mutates entries for the lost seats).
1444        if self.state.status == RunStatus::Stalled {
1445            if self.recover_stall().await? {
1446                self.finish_after_tally().await?;
1447            } else {
1448                // Still below quorum: persist the marker and stay resumable.
1449                self.state.save()?;
1450            }
1451            return Ok(());
1452        }
1453        // A run parked inside `land` - watching CI, mid fix-round, or
1454        // waiting on the owner's merge approval - resumes directly into it,
1455        // never back through `prep`. Everything before `merge` already
1456        // concluded; that is the only way `status` reaches `Landing` in the
1457        // first place. Re-walking `review_loop` first would also be actively
1458        // wrong: its own status recomputation (see its doc) treats any
1459        // clean round as reason to set `status` to `Gating`, which would
1460        // clobber this marker before `merge` ever ran, and this run would
1461        // never find its way back into `land` at all.
1462        if self.state.status == RunStatus::Landing
1463            || crate::handover::landing_checkpoint(&self.state)
1464        {
1465            self.state.land_sync_pending = true;
1466            self.state.save()?;
1467            self.run_land().await?;
1468            // `run_land` may have settled the run right here - CI came back
1469            // green and the PR merged, say - without ever passing back
1470            // through `merge`'s own trailing call. Whatever it left `status`
1471            // as is what this has to read.
1472            self.settle_questions();
1473            return Ok(());
1474        }
1475        // A run parked at `merge` on the owner's word about a withheld PR text
1476        // resumes straight into `merge`. Re-walking the tail would sync to the
1477        // base again (possibly rebasing, possibly spending the fixer) after the
1478        // gate had passed, and then push a tree nobody reviewed or gated.
1479        if self.state.github_text.is_some()
1480            && self.state.merge.is_none()
1481            && self.state.status == RunStatus::Gating
1482        {
1483            self.merge().await?;
1484            if self.resource_halt()? {
1485                return Ok(());
1486            }
1487            self.state.save()?;
1488            return Ok(());
1489        }
1490        if let Err(e) = self.prep().await {
1491            if e.is::<crate::repo_lock::ResourceBlocked>() {
1492                return Err(e);
1493            }
1494            self.state.status = RunStatus::Blocked;
1495            self.state
1496                .prep_failure
1497                .get_or_insert_with(|| format!("{e:#}"));
1498            self.state
1499                .event("prep", format!("preparation stopped: {e:#}"));
1500            self.state.save()?;
1501            return Err(e);
1502        }
1503        if self.park_here()? {
1504            return Ok(());
1505        }
1506        self.advise().await?;
1507        if self.park_here()? {
1508            return Ok(());
1509        }
1510        self.implement().await?;
1511        if self.park_here()? {
1512            return Ok(());
1513        }
1514        // `after_implement` already saved the state and settled any open
1515        // questions when it set this; nothing later in the graph has
1516        // anything to judge.
1517        if self.state.status == RunStatus::VerifiedNoop {
1518            return Ok(());
1519        }
1520        self.judge().await?;
1521        if self.park_here()? {
1522            return Ok(());
1523        }
1524        self.deliberate().await?;
1525        if self.park_here()? {
1526            return Ok(());
1527        }
1528        self.vote().await?;
1529        if self.park_here()? {
1530            return Ok(());
1531        }
1532        self.tally()?;
1533        // A verdict that lost its quorum is not trustworthy: do not review,
1534        // gate, or merge on it. Everything already done is kept, so the run
1535        // stays resumable (or foldable); the human can replace the agent that
1536        // ran out of quota and pick it up.
1537        if self.state.status == RunStatus::Stalled {
1538            // Persist the stalled marker now — the normal end-of-execute save
1539            // below is below this early return, and without it a resumed run
1540            // would reload a pre-tally status and keep going.
1541            self.state.save()?;
1542            return Ok(());
1543        }
1544        self.finish_after_tally().await?;
1545        Ok(())
1546    }
1547
1548    /// A wave could not start its agents because a shared resource stayed busy
1549    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1550    /// resumable. Unstarted seats have no result; completed peers retain
1551    /// their saved candidate state.
1552    fn stop_for_resource(&mut self) -> Result<()> {
1553        // A wait abandoned because the run is parking stays a park.
1554        if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1555        {
1556            self.state.status = RunStatus::Blocked;
1557        }
1558        self.state.save()
1559    }
1560
1561    /// Stop walking the graph if a node ended on a resource block.
1562    fn resource_halt(&mut self) -> Result<bool> {
1563        if self.state.resource_block.is_none() {
1564            return Ok(false);
1565        }
1566        self.stop_for_resource()?;
1567        Ok(true)
1568    }
1569
1570    /// Park here if asked to, recording it in the run's own timeline.
1571    ///
1572    /// Returns whether the caller should stop walking the graph. The state is
1573    /// saved either way by the node that just finished; this adds the event so
1574    /// the operator's card says why a run that is neither finished nor moving
1575    /// is sitting where it is.
1576    fn park_here(&mut self) -> Result<bool> {
1577        let parking = self.pause.parked() || self.interrupt.parked();
1578        if !parking && self.state.resource_block.is_some() {
1579            self.stop_for_resource()?;
1580            return Ok(true);
1581        }
1582        // Either handle asking is enough - see `Pause`'s own doc for why
1583        // they are never the same one. `interrupt` is checked second so a
1584        // reason it carries is preferred in the message below over a plain
1585        // shutdown park racing it at the same boundary.
1586        if !self.pause.parked() && !self.interrupt.parked() {
1587            return Ok(false);
1588        }
1589        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1590        let why = match cause.as_ref().map(ParkCause::label) {
1591            Some(reason) => format!(
1592                "parked after `{}` ({reason}) — resume to carry on from here",
1593                self.state.status.as_str()
1594            ),
1595            None => format!(
1596                "parked after `{}` — resume to carry on from here",
1597                self.state.status.as_str()
1598            ),
1599        };
1600        self.state.event("park", why);
1601        self.state
1602            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1603        self.state.parked = true;
1604        self.state.save()?;
1605        Ok(true)
1606    }
1607
1608    /// Only a new explicit operator approval may retry failed preparation.
1609    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1610        if self.state.prep_failure.is_none() {
1611            return Ok(());
1612        }
1613        let failed_at = self
1614            .state
1615            .events
1616            .iter()
1617            .rev()
1618            .find(|event| event.node == "prep")
1619            .map_or(self.state.updated_at, |event| event.at);
1620        anyhow::ensure!(
1621            approval_at > failed_at,
1622            "preparation retry requires a new explicit release after the failure"
1623        );
1624        self.state.prep_failure = None;
1625        self.state.prep_complete = Some(false);
1626        self.state
1627            .event("prep-retry", "operator approved retry of preparation");
1628        Ok(())
1629    }
1630
1631    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1632    pub fn on_pause(&mut self, pause: Pause) {
1633        self.pause = pause;
1634    }
1635
1636    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1637    /// scheduler asking this one run - and no other - to park so a task
1638    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1639    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1640    /// handle.
1641    pub fn watch_interrupt(&mut self, pause: Pause) {
1642        self.interrupt = pause;
1643    }
1644
1645    /// Abandon this run's own open questions, once `status` has actually
1646    /// settled rather than merely paused.
1647    ///
1648    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1649    /// either back up with the candidates, the review round and the seat
1650    /// sessions already on disk, so a question an implementer asked mid-round
1651    /// may still get a real answer read by a real resume. Only the statuses
1652    /// `resumable` excludes are actually final: the run merged, it reached
1653    /// `Ready` with nothing left to do, it failed outright with no
1654    /// established point to continue from, or every candidate agreed, with
1655    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1656    /// every one of those the seat that asked is gone for good, exactly like
1657    /// the run being deleted under `magi run rm` - so the same cleanup
1658    /// applies, worded for what actually happened instead of "the run was
1659    /// deleted".
1660    ///
1661    /// Best-effort and silent on success: called from every place `status`
1662    /// can land on one of those three, including ones a resumed run revisits,
1663    /// so it must cost nothing when there was nothing open to begin with.
1664    fn settle_questions(&mut self) {
1665        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1666            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1667        }
1668    }
1669
1670    /// The tail of the graph after a trustworthy tally: fold losers, review,
1671    /// gate, merge, and persist.
1672    async fn finish_after_tally(&mut self) -> Result<()> {
1673        // Before the losers are folded: their branches are what the graft
1674        // reads from.
1675        self.graft().await?;
1676        if self.resource_halt()? {
1677            return Ok(());
1678        }
1679        self.fold_losers().await?;
1680        // Before review starts, and again right before the gate: a run's
1681        // review rounds can themselves take long enough for the base to move
1682        // a second time, and the gate is the one node whose "green" gets
1683        // acted on.
1684        self.sync_to_base().await?;
1685        if self.resource_halt()? {
1686            return Ok(());
1687        }
1688        if self.state.status == RunStatus::AlreadyInBase {
1689            return Ok(());
1690        }
1691        self.review_loop().await?;
1692        if self.resource_halt()? {
1693            return Ok(());
1694        }
1695        self.sync_to_base().await?;
1696        if self.resource_halt()? {
1697            return Ok(());
1698        }
1699        if self.state.status == RunStatus::AlreadyInBase {
1700            return Ok(());
1701        }
1702        self.gate().await?;
1703        self.merge().await?;
1704        if self.resource_halt()? {
1705            return Ok(());
1706        }
1707        self.state.save()?;
1708        Ok(())
1709    }
1710
1711    // ---------------------------------------------------------------- prep
1712
1713    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1714    /// run with the step and its output in the event; it is never swallowed.
1715    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1716    /// resume starts over instead of finding candidates and missing seats.
1717    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1718        let result = match ensure_setup_config(&mut self.state, repo).await {
1719            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1720            Err(e) => Err(e),
1721        };
1722        if let Err(e) = result {
1723            git::worktree_remove_clean(repo, wt).await.ok();
1724            if in_prep {
1725                self.rollback_prep(repo).await;
1726            }
1727            self.state.status = RunStatus::Blocked;
1728            self.state.event(
1729                "setup",
1730                format!("worktree setup failed in {}: {e:#}", wt.display()),
1731            );
1732            self.state.save()?;
1733            return Err(e);
1734        }
1735        Ok(())
1736    }
1737
1738    /// Undo everything `prep` created, so a resume re-runs it whole.
1739    async fn rollback_prep(&mut self, repo: &Path) {
1740        let root = self.state.worktree_root();
1741        for c in std::mem::take(&mut self.state.candidates) {
1742            if matches!(
1743                git::worktree_remove_clean(repo, &c.worktree).await,
1744                Ok(true)
1745            ) {
1746                git::branch_delete(repo, &c.branch).await.ok();
1747            } else {
1748                self.state.candidates.push(c);
1749            }
1750        }
1751        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1752        for j in 1..=seats {
1753            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1754                .await
1755                .ok();
1756        }
1757        for k in 1..=self.state.config.graph.advisors {
1758            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1759                .await
1760                .ok();
1761        }
1762    }
1763
1764    async fn prep(&mut self) -> Result<()> {
1765        if let Some(reason) = &self.state.prep_failure {
1766            bail!(
1767                "candidate preparation previously failed: {reason}; correct the source before retrying"
1768            );
1769        }
1770        if self
1771            .state
1772            .prep_complete
1773            .unwrap_or(!self.state.candidates.is_empty())
1774        {
1775            return Ok(());
1776        }
1777        let repo = self.state.repo.clone();
1778        // Includes rollback, config, refs, checkout creation and setup. No
1779        // repository mutation may precede the real shared lock acquisition.
1780        let _coordination = self.coordinate(&repo).await?;
1781        let base = self.state.base_commit.clone();
1782        let plan = self.state.sources.plan(&repo, &base).await?;
1783        self.rollback_prep(&repo).await;
1784        if !self.state.candidates.is_empty() {
1785            bail!(
1786                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1787            );
1788        }
1789        self.state.prep_complete = Some(false);
1790        self.state.status = RunStatus::Prep;
1791        self.state.save()?;
1792        let repo = self.state.repo.clone();
1793        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1794        let root = self.state.worktree_root();
1795        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1796
1797        // The hook is the write-time half of the blindness contract; the
1798        // presentation filter in `blind` is the half that cannot be bypassed.
1799        let hooks_dir = self.state.dir().join("hooks");
1800        if self.state.config.blind.commit_msg_hook {
1801            std::fs::create_dir_all(&hooks_dir)
1802                .with_context(|| format!("create {}", hooks_dir.display()))?;
1803            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1804            let path = hooks_dir.join("commit-msg");
1805            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1806            make_executable(&path)?;
1807            // Ref-counted rather than a plain idempotent set: with more than
1808            // one run able to be in flight in the same repository at once
1809            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1810            // true?" check cannot tell "another run of mine still needs
1811            // this" from "nobody does", and the run that happens to finish
1812            // first would disable the hook out from under a sibling still
1813            // relying on it.
1814            git::acquire_worktree_config(&repo).await?;
1815            self.state.enabled_worktree_config = true;
1816        }
1817
1818        // `[worktree] setup` hides its products per worktree, which needs the
1819        // same per-worktree config the hook does. Held until fold, like it.
1820        ensure_setup_config(&mut self.state, &repo).await?;
1821
1822        for (index, (spec, label)) in self
1823            .roles
1824            .implementers
1825            .clone()
1826            .into_iter()
1827            .zip(labels)
1828            .enumerate()
1829        {
1830            let branch = self.state.branch_for(label);
1831            let worktree = root.join(format!("cand-{label}"));
1832            // Persist intent before Git can leave a partially created tree.
1833            // Candidate registration follows setup, so it cannot inventory
1834            // crashes or refusals during creation itself.
1835            self.state.event(
1836                "prep-intent",
1837                serde_json::json!({
1838                    "branch": branch,
1839                    "worktree": worktree,
1840                    "start": start,
1841                    "picks": plan.picks,
1842                })
1843                .to_string(),
1844            );
1845            self.state.save()?;
1846            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1847            if self.state.config.blind.commit_msg_hook {
1848                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1849            }
1850            git::local_exclude(&worktree, "/.magi/").await?;
1851            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1852                // Nothing of this prep may survive: a resume must start over,
1853                // not find half the candidates and skip the rest.
1854                git::worktree_remove_clean(&repo, &worktree).await.ok();
1855                git::branch_delete(&repo, &branch).await.ok();
1856                self.rollback_prep(&repo).await;
1857                self.state.status = RunStatus::Blocked;
1858                self.state
1859                    .event("prep", format!("worktree setup failed: {e:#}"));
1860                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1861                self.state.save()?;
1862                return Err(e);
1863            }
1864            self.state.candidates.push(Candidate {
1865                index,
1866                label,
1867                agent: spec.id.clone(),
1868                branch,
1869                worktree,
1870                summary: String::new(),
1871                stat: String::new(),
1872                files: 0,
1873                commits: 0,
1874                empty: false,
1875                failed: None,
1876                verified_noop: None,
1877                duration_ms: 0,
1878                folded: false,
1879            });
1880            self.state.save()?;
1881            for pick in &plan.picks {
1882                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1883                {
1884                    self.state.status = RunStatus::Blocked;
1885                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1886                    self.state
1887                        .event("prep", format!("cannot apply referenced commit: {e}"));
1888                    self.state.save()?;
1889                    return Err(e);
1890                }
1891            }
1892        }
1893
1894        for j in 1..=self.roles.judges.len() {
1895            let wt = root.join(format!("judge-{j}"));
1896            if !wt.exists() {
1897                git::worktree_add_detached(&repo, &wt, &base).await?;
1898                self.setup_seat_worktree(&repo, &wt, true).await?;
1899            }
1900        }
1901
1902        // Disposable, detached checkouts for the design-deliberation stage's
1903        // advisor seats — the same shape as the judges' above, at the same
1904        // base commit, since advisors also only ever read. Sized off the
1905        // configured count directly rather than a resolved roster: unlike
1906        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1907        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1908        // `prep` has no `ResolvedRoles` field to read a count from here.
1909        if self.state.config.graph.advise {
1910            for k in 1..=self.state.config.graph.advisors {
1911                let wt = root.join(format!("advisor-{k}"));
1912                if !wt.exists() {
1913                    git::worktree_add_detached(&repo, &wt, &base).await?;
1914                    self.setup_seat_worktree(&repo, &wt, true).await?;
1915                }
1916            }
1917        }
1918
1919        // A judge cannot tell it is looking at its own patch — the seats keep
1920        // separate conversations — but a panel that shares agents with the
1921        // field is less independent than it looks, and that is worth saying out
1922        // loud once per run rather than leaving it in the config.
1923        let authors: Vec<&str> = self
1924            .roles
1925            .implementers
1926            .iter()
1927            .map(|a| a.id.as_str())
1928            .collect();
1929        let overlap: Vec<String> = self
1930            .roles
1931            .judges
1932            .iter()
1933            .enumerate()
1934            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1935            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1936            .collect();
1937        if !overlap.is_empty() {
1938            let note = format!(
1939                "{} also authored a candidate; blind, but the panel is less \
1940                 independent than {} distinct agents would be",
1941                overlap.join(", "),
1942                self.roles.judges.len()
1943            );
1944            self.state.event("prep", note);
1945        }
1946
1947        self.state.event(
1948            "prep",
1949            format!(
1950                "{} candidates, {} judges, base {} ({})",
1951                self.state.candidates.len(),
1952                self.roles.judges.len(),
1953                &self.state.base_commit[..7.min(self.state.base_commit.len())],
1954                self.state.base_branch
1955            ),
1956        );
1957        self.state.prep_complete = Some(true);
1958        self.state.status = RunStatus::Implementing;
1959        self.state.save()?;
1960        Ok(())
1961    }
1962
1963    // -------------------------------------------------------------- advise
1964
1965    /// The design-deliberation stage: independent, read-only advisor seats
1966    /// each sketch a design before any implementer touches the repository,
1967    /// and (when at least one produced a usable proposal) a synthesis seat
1968    /// blends them into a brief `implement` carries in every candidate's
1969    /// prompt.
1970    ///
1971    /// `[graph] advise` is the on/off switch, on by default; `[graph]
1972    /// advisors` is the proposal count. Everything here is best-effort and
1973    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
1974    /// that cannot reach quota, or a synthesis seat that produced nothing
1975    /// usable all leave `implement` exactly as it was before this stage
1976    /// existed — the task instruction alone — rather than failing the whole
1977    /// competition over an enrichment stage. Every outcome is still recorded
1978    /// as an event, so a run that got nothing from this stage says why.
1979    ///
1980    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
1981    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
1982    /// resumed run whose stage failed would re-run it, and re-spend the
1983    /// agent calls, on every reentry before `implement`.
1984    ///
1985    /// Also skipped once any candidate shows implementation progress — the
1986    /// exact predicate `implement` itself uses to decide a candidate is no
1987    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
1988    /// is not enough: a run created by an older binary that predates this
1989    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
1990    /// an already-`Implementing`-or-later run under this build would
1991    /// otherwise walk straight back through `prep` (a no-op once candidates
1992    /// exist) into this node and spawn every advisor seat against worktrees
1993    /// `prep` never recreated — after implementation has already started,
1994    /// which is exactly the invariant this stage exists to guarantee.
1995    async fn advise(&mut self) -> Result<()> {
1996        let implement_untouched = self
1997            .state
1998            .candidates
1999            .iter()
2000            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2001        if !self.state.config.graph.advise || self.state.advise_attempted {
2002            return Ok(());
2003        }
2004        if !implement_untouched {
2005            self.state.event(
2006                "advise",
2007                "skipping the design-deliberation stage: at least one \
2008                 candidate already shows implementation progress, so this \
2009                 run is past the point the stage exists to run before"
2010                    .to_owned(),
2011            );
2012            self.state.advise_attempted = true;
2013            self.state.save()?;
2014            return Ok(());
2015        }
2016        let run_id = self.state.id.clone();
2017        let prompts = self.state.config.prompts.clone();
2018        let instruction = self.state.instruction.clone();
2019        let language = self.state.config.graph.language.clone();
2020        let root = self.state.worktree_root();
2021        let n = self.state.config.graph.advisors;
2022        let where_recorded = self.state.dir().join("run.json");
2023
2024        let seats = match self.state.config.advisors() {
2025            Ok(seats) if !seats.is_empty() => seats,
2026            Ok(_) => {
2027                self.state.event(
2028                    "advise",
2029                    format!(
2030                        "[graph] advisors is 0; skipping the design-deliberation \
2031                         stage and continuing without a synthesis brief (see {})",
2032                        where_recorded.display()
2033                    ),
2034                );
2035                self.state.advise_attempted = true;
2036                self.state.save()?;
2037                return Ok(());
2038            }
2039            Err(e) => {
2040                self.state.event(
2041                    "advise",
2042                    format!(
2043                        "could not resolve advisor seats ({e:#}); continuing \
2044                         without a design-deliberation brief (see {})",
2045                        where_recorded.display()
2046                    ),
2047                );
2048                self.state.advise_attempted = true;
2049                self.state.save()?;
2050                return Ok(());
2051            }
2052        };
2053
2054        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2055        let artifacts = agent::artifacts_dir(&self.state.dir());
2056        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2057
2058        let mut jobs = Vec::new();
2059        for (i, spec) in seats.iter().cloned().enumerate() {
2060            let seat_key = format!("advisor-{}", i + 1);
2061            let seat = self.seat(&seat_key, &spec.id);
2062            jobs.push(SeatJob {
2063                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2064                spec,
2065                seat,
2066                cwd: worktrees[i % worktrees.len()].clone(),
2067                timeout,
2068                allow_write: false,
2069                sessions: false,
2070                artifacts: artifacts.clone(),
2071                stem: seat_key,
2072                handover: None,
2073            });
2074        }
2075
2076        self.state.event(
2077            "advise",
2078            format!(
2079                "{} advisor seat(s) sketching a design in parallel",
2080                jobs.len()
2081            ),
2082        );
2083        let mut quota_losses = Vec::new();
2084        let cache = self.state.config.cache_dir();
2085        let ctx = WaveCtx {
2086            carry_seats: false,
2087            pauses: Some((&self.pause, &self.interrupt)),
2088            run: &run_id,
2089            node: "advise",
2090            prompts: &prompts,
2091            cache: cache.as_deref(),
2092            round: None,
2093        };
2094        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2095        let results = ask_json_wave::<Proposal>(
2096            jobs,
2097            Arc::clone(&self.sem),
2098            self.state.config.graph.retries,
2099            &advisor_roster,
2100            &ctx,
2101            &mut quota_losses,
2102            &mut self.state,
2103            &|p: &Proposal| p.validate(),
2104        )
2105        .await;
2106        if self.state.resource_block.is_some() {
2107            return self.stop_for_resource();
2108        }
2109        self.state.quota.extend(quota_losses);
2110
2111        let mut records = Vec::with_capacity(results.len());
2112        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2113            let agent_id = seat.agent.clone();
2114            self.state.seats.insert(seat.key.clone(), seat);
2115            match res {
2116                Ok((proposal, out)) => {
2117                    self.state
2118                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2119                    records.push(advise::AdvisorRecord::proposed(
2120                        i + 1,
2121                        agent_id,
2122                        proposal,
2123                        out.duration_ms,
2124                    ));
2125                }
2126                Err(e) => {
2127                    self.state.event(
2128                        "advise",
2129                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2130                    );
2131                    records.push(advise::AdvisorRecord::failed(
2132                        i + 1,
2133                        agent_id,
2134                        e.to_string(),
2135                    ));
2136                }
2137            }
2138        }
2139
2140        let mut advice = advise::Advice {
2141            records,
2142            synthesis: None,
2143        };
2144        if advice.proposals().is_empty() {
2145            self.state.event(
2146                "advise",
2147                "no advisor produced a usable proposal; continuing without a \
2148                 synthesis brief"
2149                    .to_owned(),
2150            );
2151        } else {
2152            match self
2153                .synthesize_brief(
2154                    &advice,
2155                    &instruction,
2156                    &language,
2157                    &worktrees[0],
2158                    &artifacts,
2159                    &run_id,
2160                    &prompts,
2161                    cache.as_deref(),
2162                )
2163                .await
2164            {
2165                Ok(Some(text)) => {
2166                    self.state.event(
2167                        "advise",
2168                        "synthesized a design brief for the implementer".to_owned(),
2169                    );
2170                    advice.synthesis = Some(text);
2171                }
2172                Ok(None) => {
2173                    self.state.event(
2174                        "advise",
2175                        "the synthesis seat produced nothing usable; continuing \
2176                         without a design brief"
2177                            .to_owned(),
2178                    );
2179                }
2180                Err(e) => {
2181                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2182                        return Err(e);
2183                    }
2184                    self.state.event(
2185                        "advise",
2186                        format!("could not synthesize a design brief: {e:#}"),
2187                    );
2188                }
2189            }
2190        }
2191        advise::apply_reflection(&mut advice);
2192
2193        self.state.advice = Some(advice);
2194        self.state.advise_attempted = true;
2195        self.state.save()?;
2196        Ok(())
2197    }
2198
2199    /// The synthesis seat: reads every advisor's proposal and blends them
2200    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2201    /// out of [`Runner::advise`] only for readability — it is not called
2202    /// anywhere else.
2203    ///
2204    /// Picked the same way [`crate::talk`]'s standing conversation and
2205    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2206    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2207    /// order (a claude seat, else the first runnable agent in roster order)
2208    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2209    /// in [`crate::config`] for why a dedicated field exists here at all.
2210    #[allow(clippy::too_many_arguments)]
2211    async fn synthesize_brief(
2212        &mut self,
2213        advice: &advise::Advice,
2214        instruction: &str,
2215        language: &str,
2216        cwd: &Path,
2217        artifacts: &Path,
2218        run_id: &str,
2219        prompts: &Prompts,
2220        cache: Option<&Path>,
2221    ) -> Result<Option<String>> {
2222        let chain = agent::pick_chain(
2223            &self.state.config.agents,
2224            self.state.config.roles.synthesizer.as_ref(),
2225            &agent::installed,
2226            "synthesizer",
2227        )?;
2228        let proposals = advice.proposals();
2229        let mut prompt = prompt::with_overlay(
2230            prompt::synthesize_brief(instruction, &proposals, language),
2231            prompts.overlay("advise"),
2232        );
2233        if cache.is_some() {
2234            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2235            // below — see `prompt::build_cache_note`'s doc for why telling a
2236            // read-only seat to build through the shared cache is exactly how
2237            // a sandbox's write refusal gets misread as a defect.
2238            prompt.push('\n');
2239            prompt.push_str(&prompt::build_cache_note("advise", false));
2240        }
2241        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2242        // Each id is tried once, in order; a quota hit, error or unusable
2243        // answer moves to the next. The seat is single-turn (`sessions:
2244        // false`) and the prompt is the whole context, so a fallback agent
2245        // needs nothing carried over.
2246        let mut last = None;
2247        for (n, spec) in chain.iter().enumerate() {
2248            if n > 0 {
2249                self.state
2250                    .event("advise", format!("synthesis falling back to {}", spec.id));
2251            }
2252            let mut seat = self.seat("advise-synthesis", &spec.id);
2253            let wait = self.repository_wait();
2254            let outcome = wait
2255                .scope(agent::invoke(
2256                    spec,
2257                    &mut seat,
2258                    &Invocation {
2259                        scope: crate::agent::InvocationScope::Repository,
2260                        coordination: self.state.config.coordination.as_ref(),
2261                        fetched_base: None,
2262                        cwd,
2263                        prompt: &prompt,
2264                        timeout,
2265                        allow_write: false,
2266                        unsandboxed: false,
2267                        sessions: false,
2268                        artifacts,
2269                        stem: &if n == 0 {
2270                            "advise-synthesis".to_owned()
2271                        } else {
2272                            format!("advise-synthesis-{}", spec.id)
2273                        },
2274                        run: run_id,
2275                        node: "advise",
2276                        cache_dir: None,
2277                        attachments: &[],
2278                        writable: &[],
2279                    },
2280                ))
2281                .await;
2282            if outcome.is_ok() {
2283                self.state.seats.insert(seat.key.clone(), seat);
2284            }
2285            let advance = agent::chain_advances(&outcome);
2286            last = Some(outcome);
2287            if !advance {
2288                break;
2289            }
2290        }
2291        // Exhausted: the last attempt's result is what a single failed seat
2292        // would have produced.
2293        let out = last.expect("a chain holds at least one agent")?;
2294        if !out.usable() {
2295            return Ok(None);
2296        }
2297        let text =
2298            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2299        Ok((!text.trim().is_empty()).then_some(text))
2300    }
2301
2302    // ----------------------------------------------------------- implement
2303
2304    async fn implement(&mut self) -> Result<()> {
2305        self.persist_candidates().await?;
2306        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2307        // agent files with `magi task add` name the run that paid for it. The
2308        // prompt overlay is cloned alongside it because the waves borrow it
2309        // while `self` is mutably borrowed by the node's own bookkeeping.
2310        let run_id = self.state.id.clone();
2311        let prompts = self.state.config.prompts.clone();
2312        let todo: Vec<usize> = self
2313            .state
2314            .candidates
2315            .iter()
2316            .enumerate()
2317            .filter(|(_, c)| {
2318                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2319                    && c.failed.is_none()
2320                    && !c.empty
2321            })
2322            .map(|(i, _)| i)
2323            .collect();
2324        if todo.is_empty() {
2325            return self.after_implement();
2326        }
2327        self.state.status = RunStatus::Implementing;
2328
2329        let language = self.state.config.graph.language.clone();
2330        let github_language = self.state.config.graph.github_language.clone();
2331        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2332        let sessions = self.state.config.graph.sessions;
2333        let artifacts = agent::artifacts_dir(&self.state.dir());
2334        // The design-deliberation stage's blended brief, when `advise` found
2335        // one — carried into every implementer's prompt the same way
2336        // regardless of which candidate it is.
2337        let brief = self
2338            .state
2339            .advice
2340            .as_ref()
2341            .and_then(|a| a.synthesis.as_deref())
2342            .map(str::to_owned);
2343        let attachments = self.state.attachments.clone();
2344
2345        let mut jobs = Vec::new();
2346        for &i in &todo {
2347            let (index, label, worktree) = {
2348                let c = &self.state.candidates[i];
2349                (c.index, c.label, c.worktree.clone())
2350            };
2351            let spec = self.roles.implementers[index].clone();
2352            let seat_key = format!("impl-{label}");
2353            let seat = self.seat(&seat_key, &spec.id);
2354            let instruction = seeded_instruction(&self.state);
2355            jobs.push(SeatJob {
2356                spec,
2357                seat,
2358                prompt: prompt::implement_in(
2359                    &instruction,
2360                    &worktree.to_string_lossy(),
2361                    &language,
2362                    &github_language,
2363                    brief.as_deref(),
2364                    &attachments,
2365                ),
2366                cwd: worktree,
2367                timeout,
2368                allow_write: true,
2369                sessions,
2370                artifacts: artifacts.clone(),
2371                stem: format!("impl-{label}"),
2372                handover: None,
2373            });
2374        }
2375
2376        self.state.event(
2377            "implement",
2378            format!("{} candidates in parallel", jobs.len()),
2379        );
2380        // Kept so a seat whose CLI hung up can be asked again from the same
2381        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2382        // can update a seat's own entry once a fallback agent takes it over —
2383        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2384        // whichever agent actually answered, not the one that quota'd out.
2385        let mut sent = jobs.clone();
2386        let cache = self.state.config.cache_dir();
2387        let ctx = WaveCtx {
2388            carry_seats: false,
2389            pauses: Some((&self.pause, &self.interrupt)),
2390            run: &run_id,
2391            node: "implement",
2392            prompts: &prompts,
2393            cache: cache.as_deref(),
2394            round: None,
2395        };
2396        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2397        if !wave_blocked(&results) {
2398            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2399                .await;
2400            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2401                .await?;
2402            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2403                .await;
2404        }
2405        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2406
2407        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2408            && results
2409                .iter()
2410                .any(|(_, _, out)| FailClass::of(out).is_some());
2411        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2412            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2413                // Executed turns interrupted by disk pressure already carry
2414                // a preservation barrier from wave. Admission-only waits do
2415                // not. Keep the interrupted seat's session for recovery too.
2416                if self
2417                    .state
2418                    .candidate_preservation
2419                    .contains_key(&self.state.candidates[i].label)
2420                {
2421                    self.state.seats.insert(seat.key.clone(), seat);
2422                }
2423                continue;
2424            }
2425            let seat_key = seat.key.clone();
2426            // A quota fallback (`resume_seat_handovers`) may have handed this
2427            // seat to a different agent than the one `prep` recorded on the
2428            // candidate; the stats tables and any later fixer-defaults-to-
2429            // winner's-author lookup must credit whoever actually answered —
2430            // unless every fallback also quota'd out, in which case nobody
2431            // actually answered and crediting the last agent tried would
2432            // erase every earlier agent's own quota loss from the stats
2433            // tables instead of just this one seat's.
2434            let agent = seat.agent.clone();
2435            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2436            self.state.seats.insert(seat.key.clone(), seat);
2437            let label = self.state.candidates[i].label;
2438            self.state.interrupted_implementations.remove(&label);
2439
2440            let (summary, duration, failed, verified_claim) = match out {
2441                AgentOutcome::Ok(o) => {
2442                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2443                    let failed = (!o.usable()).then(|| {
2444                        if o.timed_out {
2445                            "agent timed out".to_owned()
2446                        } else {
2447                            format!(
2448                                "agent exited with {}",
2449                                crate::notices::exit_status(o.exit_code)
2450                            )
2451                        }
2452                    });
2453                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2454                    (text, o.duration_ms, failed, verified_claim)
2455                }
2456                // Left un-resumed by `resume_undelivered` (a dirty tree
2457                // already rescues the work, or there was no session left to
2458                // resume into) — reported like the ordinary failure it is,
2459                // never as if `o.text` (the CLI's raw error JSON) were an
2460                // answer.
2461                AgentOutcome::Dropped(o) => {
2462                    let why = o
2463                        .dropped
2464                        .as_ref()
2465                        .map(|d| d.why.as_str())
2466                        .unwrap_or("the CLI ended the stream without delivering its answer");
2467                    (
2468                        String::new(),
2469                        o.duration_ms,
2470                        Some(format!("the CLI dropped the stream ({why})")),
2471                        None,
2472                    )
2473                }
2474                AgentOutcome::Quota(o) => {
2475                    self.state.quota.push(QuotaLoss {
2476                        seat: seat_key,
2477                        node: "implement".to_owned(),
2478                        at: Timestamp::now(),
2479                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2480                    });
2481                    (
2482                        String::new(),
2483                        o.duration_ms,
2484                        Some("rate limited (quota); produced no change".to_owned()),
2485                        None,
2486                    )
2487                }
2488                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2489                    (String::new(), 0, Some(e), None)
2490                }
2491            };
2492
2493            // Save every completed turn before any fallible Git operation. A failed
2494            // rescue must not erase the reply or prevent collection of other seats.
2495            let c = &mut self.state.candidates[i];
2496            if !exhausted_the_fallback_chain {
2497                c.agent = agent;
2498            }
2499            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2500            c.duration_ms = duration;
2501            c.failed = failed;
2502            c.verified_noop = verified_claim;
2503            self.state
2504                .candidate_preservation
2505                .insert(label, "awaiting persistence".to_owned());
2506        }
2507        self.state.save()?;
2508        self.persist_candidates().await?;
2509
2510        if resource_blocked {
2511            return self.stop_for_resource();
2512        }
2513        if selected_failed {
2514            self.state.status = RunStatus::Blocked;
2515            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2516            self.state.save()?;
2517            bail!(
2518                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2519            );
2520        }
2521        self.after_implement()
2522    }
2523
2524    /// Retry only persistence after a completed implementation turn.
2525    async fn persist_candidates(&mut self) -> Result<()> {
2526        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2527        for label in labels {
2528            let i = self
2529                .state
2530                .candidates
2531                .iter()
2532                .position(|c| c.label == label)
2533                .context("pending candidate is missing")?;
2534            let c = self.state.candidates[i].clone();
2535            let interrupted = self.state.interrupted_implementations.contains(&label);
2536            let result = async {
2537                let wait = crate::repo_lock::WaitPolicy::new(
2538                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2539                    || false,
2540                );
2541                let _coordination =
2542                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2543                // Best effort evidence requires no index writes. A successful snapshot
2544                // never authorizes removal of a tree whose rescue failed.
2545                if let Err(e) = git::snapshot_uncommitted(
2546                    &c.worktree,
2547                    &self
2548                        .state
2549                        .dir()
2550                        .join("artifacts")
2551                        .join(format!("candidate-{label}-recovery")),
2552                )
2553                .await
2554                {
2555                    self.state.event(
2556                        "preserve",
2557                        format!("candidate {label} snapshot incomplete: {e:#}"),
2558                    );
2559                }
2560                let rescued = git::rescue_commit(
2561                    &c.worktree,
2562                    &format!("magi: candidate {label} (uncommitted work)"),
2563                )
2564                .await?;
2565                self.state.note_withheld("implement", &rescued.withheld);
2566                let base = &self.state.base_commit;
2567                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2568                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2569                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2570                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2571                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2572                if c.empty && commits > 0 && !patch.trim().is_empty()
2573                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2574                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2575                        || self.state.tally.is_some())
2576                {
2577                    // Keep the old classification and pending marker so every
2578                    // retry stops too. Retain decisions as recovery evidence;
2579                    // only a supported recovery operation may replace them.
2580                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2581                }
2582                let c = &mut self.state.candidates[i];
2583                c.commits = commits;
2584                c.stat = stat;
2585                c.files = files;
2586                let no_changes = commits == 0 || patch.trim().is_empty();
2587                c.empty = no_changes && !interrupted;
2588                if interrupted && no_changes {
2589                    c.failed = None;
2590                    c.verified_noop = None;
2591                }
2592                if !c.empty {
2593                    c.failed = None;
2594                    c.verified_noop = None;
2595                }
2596                self.state.event(
2597                    "preserve",
2598                    format!("candidate {label}: {files} files, {commits} commits"),
2599                );
2600                Ok::<_, anyhow::Error>(())
2601            }
2602            .await;
2603            match result {
2604                Ok(()) => {
2605                    self.state.candidate_preservation.remove(&label);
2606                    if !interrupted || self.state.candidates[i].files > 0 {
2607                        self.state.interrupted_implementations.remove(&label);
2608                    }
2609                }
2610                Err(e) => {
2611                    let reason = format!("candidate {label} preservation failed: {e:#}");
2612                    self.state.event("preserve", &reason);
2613                    self.state.candidate_preservation.insert(label, reason);
2614                }
2615            }
2616            if self.state.candidate_preservation.is_empty()
2617                && self.state.status == RunStatus::Blocked
2618                && self.state.candidates.iter().all(|c| c.failed.is_none())
2619            {
2620                // Persist the recovered phase with the cleared pending marker;
2621                // a crash between saves must not turn this into a selected-seat
2622                // failure hold on the next resume.
2623                self.state.status = RunStatus::Implementing;
2624                self.state.event(
2625                    "preserve",
2626                    "candidate persistence recovered; implementation can continue",
2627                );
2628            }
2629            self.state.save()?;
2630        }
2631        if !self.state.candidate_preservation.is_empty() {
2632            self.state.status = RunStatus::Blocked;
2633            self.state.save()?;
2634            bail!(
2635                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2636            );
2637        }
2638        Ok(())
2639    }
2640
2641    /// Ask again, once, for work a CLI did and then failed to hand over.
2642    ///
2643    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2644    /// status with an empty response and a usage report showing output tokens,
2645    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2646    /// seven minutes and 14,267 output tokens that arrived as an empty
2647    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2648    ///
2649    /// Two conditions, and both matter:
2650    ///
2651    /// - **Only when the tree is untouched.** Often the agent has already
2652    ///   written its files and only the closing message was lost; the rescue
2653    ///   commit below picks that up and there is nothing to ask for. Re-asking
2654    ///   then would pay for a second implementation of work already on disk.
2655    /// - **Once.** A CLI that drops one stream can drop the next, and this
2656    ///   node is the most expensive in the graph.
2657    ///
2658    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2659    /// dropped reply still carried its `conversation_id`, so the seat is asked
2660    /// to finish what it was doing rather than sent the whole task again. It
2661    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2662    /// node's - for the same reason a re-ranked judge does: restating finished
2663    /// work is not the work.
2664    ///
2665    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2666    /// same way until it resets, while an abandoned conversation is still
2667    /// there to be picked up.
2668    async fn resume_undelivered(
2669        &mut self,
2670        results: &mut [(usize, SeatState, AgentOutcome)],
2671        sent: &[SeatJob],
2672        prompts: &Prompts,
2673        run_id: &str,
2674    ) {
2675        for (wi, seat, out) in results.iter_mut() {
2676            let Some(dropped) = (match &*out {
2677                AgentOutcome::Dropped(o) => o.dropped.clone(),
2678                _ => None,
2679            }) else {
2680                continue;
2681            };
2682            let Some(job) = sent.get(*wi) else { continue };
2683            // Already on disk? Then only the closing message was lost.
2684            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2685                self.state.event(
2686                    "implement",
2687                    format!(
2688                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2689                         work is in the tree",
2690                        seat.key, dropped.output_tokens, dropped.why
2691                    ),
2692                );
2693                continue;
2694            }
2695            // The re-ask only makes sense as a resume: `resume_after_drop`
2696            // says nothing about the task, trusting the seat to still hold it.
2697            // Without a session to resume — sessions disabled, or this CLI's
2698            // drop shape happened not to carry a session id — that prompt
2699            // would open a brand-new conversation with no context at all,
2700            // which is worse than leaving this as the ordinary failure it
2701            // already is.
2702            if !has_context(&job.spec, seat, job.sessions) {
2703                self.state.event(
2704                    "implement",
2705                    format!(
2706                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2707                         is no session left to resume",
2708                        seat.key, dropped.output_tokens, dropped.why
2709                    ),
2710                );
2711                continue;
2712            }
2713            self.state.event(
2714                "implement",
2715                format!(
2716                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2717                     conversation",
2718                    seat.key, dropped.output_tokens, dropped.why
2719                ),
2720            );
2721            let mut retry = job.clone();
2722            retry.seat = seat.clone();
2723            retry.prompt = prompt::resume_after_drop(&dropped.why);
2724            retry.timeout = retry_budget(job.timeout, true);
2725            retry.stem = format!("{}-resume", job.stem);
2726            let cache = self.state.config.cache_dir();
2727            let ctx = WaveCtx {
2728                carry_seats: false,
2729                pauses: Some((&self.pause, &self.interrupt)),
2730                run: run_id,
2731                node: "implement",
2732                prompts,
2733                cache: cache.as_deref(),
2734                round: None,
2735            };
2736            let (resumed_seat, resumed) =
2737                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2738            *seat = resumed_seat;
2739            *out = resumed;
2740        }
2741    }
2742
2743    /// Fall an implement seat through to the next untried agent in the
2744    /// implementer roster when it lost to quota — or, since the handover was
2745    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2746    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2747    /// quota path itself being unchanged — instead of leaving the
2748    /// seat's loss final the moment one agent's account runs dry.
2749    ///
2750    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2751    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2752    /// truncates to the single slot rotation picked, so a solo task whose one
2753    /// implementer hits quota mid-run used to have nothing else to try. This
2754    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2755    /// unrotated roster — which is the only place the *other* candidates in
2756    /// the machine's roster still exist once `implementers` has been cut down
2757    /// to size.
2758    ///
2759    /// Walks forward from just past the seat's own original position in the
2760    /// roster, never wrapping back to the front: a later candidate slot (say
2761    /// `beta`, the roster's second entry) must fall through to the *next*
2762    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2763    /// almost certainly a different candidate's own agent already — and once
2764    /// the roster's tail is exhausted there is nothing left to fall through
2765    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2766    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2767    /// let this retry that id forever. The loop keeps falling through until
2768    /// an attempt lands something other than `Quota` or the roster's tail
2769    /// runs out of untried ids, at which point the seat is left exactly as
2770    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2771    /// `QuotaLoss` recorded, the candidate failed/empty.
2772    ///
2773    /// `sent` is taken mutably and updated with the fallback agent's spec:
2774    /// `resume_unconfirmed_commands`, which runs after this and also reads
2775    /// `sent`, must see whichever agent actually ended up answering the seat
2776    /// — reading the stale, original spec there would check session
2777    /// eligibility against the wrong CLI and could hand a fallback agent's
2778    /// session id to the agent that just lost the seat to quota.
2779    ///
2780    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2781    /// — `self.seat` only reuses state when the agent id is unchanged, so
2782    /// handing it a different id already gets this for free. Reusing the old
2783    /// seat would resume a different CLI's session as if it were a
2784    /// continuation of this one.
2785    ///
2786    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2787    /// a quota loss cuts an agent off mid-turn, so anything already in the
2788    /// tree is unfinished work, not a completed candidate a re-ask would pay
2789    /// for twice. A dirty tree is rescued into a commit first (the same
2790    /// neutral-identity rescue `implement`'s own outcome loop gives every
2791    /// candidate) so the next agent starts clean.
2792    ///
2793    /// The new agent gets the implementer's full prompt and full
2794    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2795    /// it has no session and no context, and is implementing the task from
2796    /// nothing, unlike a resumed drop which is only restating work already
2797    /// done.
2798    ///
2799    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2800    /// `implement` event, never into `self.state.quota` — that is what
2801    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2802    /// go unspent, and a seat that ultimately recovered on its second or
2803    /// third agent is not the stalled panel that check exists to catch. Only
2804    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2805    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2806    /// outcome loop already has — this helper never pushes to it itself.
2807    async fn resume_seat_handovers(
2808        &mut self,
2809        results: &mut [(usize, SeatState, AgentOutcome)],
2810        sent: &mut [SeatJob],
2811        prompts: &Prompts,
2812        run_id: &str,
2813    ) -> Result<()> {
2814        let instruction = seeded_instruction(&self.state);
2815        let language = self.state.config.graph.language.clone();
2816        let github_language = self.state.config.graph.github_language.clone();
2817        let brief = self
2818            .state
2819            .advice
2820            .as_ref()
2821            .and_then(|a| a.synthesis.as_deref())
2822            .map(str::to_owned);
2823        let attachments = self.state.attachments.clone();
2824        for (wi, seat, out) in results.iter_mut() {
2825            // Who holds the other candidate seats of this wave right now
2826            // (earlier handovers already written back to `sent`).
2827            let others: BTreeSet<String> = sent
2828                .iter()
2829                .enumerate()
2830                .filter(|(j, _)| j != wi)
2831                .map(|(_, j)| j.spec.id.clone())
2832                .collect();
2833            let Some(job) = sent.get_mut(*wi) else {
2834                continue;
2835            };
2836            // Where the seat's own original agent sits in the roster — the
2837            // fallback walk starts just past here, never at the front, so a
2838            // later candidate slot's quota loss does not fall back onto an
2839            // earlier slot's own agent.
2840            let start = self
2841                .roles
2842                .implementer_roster
2843                .iter()
2844                .position(|s| s.id == job.spec.id)
2845                .unwrap_or(0);
2846            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2847            let mut fallback_attempt = 0usize;
2848            let mut prev: Option<FailClass> = None;
2849            while let Some(cur) = FailClass::of(&*out) {
2850                if !should_hand_over(prev.as_ref(), &cur) {
2851                    break;
2852                }
2853                let Some(next) =
2854                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2855                        .cloned()
2856                else {
2857                    break;
2858                };
2859                tried.insert(next.id.clone());
2860                fallback_attempt += 1;
2861
2862                let coordination = self.coordinate(&job.cwd).await?;
2863                if let Ok(r) = git::rescue_commit(
2864                    &job.cwd,
2865                    &format!(
2866                        "magi: candidate {} (uncommitted work before {} fallback)",
2867                        seat.key,
2868                        if cur == FailClass::Quota {
2869                            "quota"
2870                        } else {
2871                            "handover"
2872                        }
2873                    ),
2874                )
2875                .await
2876                {
2877                    self.state.note_withheld("implement", &r.withheld);
2878                }
2879                drop(coordination);
2880
2881                record_handover(
2882                    &mut self.state,
2883                    "implement",
2884                    &seat.key,
2885                    &seat.agent,
2886                    &next.id,
2887                    &cur,
2888                    &fail_reason(&*out),
2889                );
2890                prev = Some(cur.clone());
2891
2892                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2893                self.state.seats.insert(seat.key.clone(), new_seat.clone());
2894                // Kept in sync on `sent` itself, not just the local retry: a
2895                // later helper (`resume_unconfirmed_commands`) reads `sent`
2896                // after this one returns and must see whichever agent is now
2897                // occupying the seat, not the one that just quota'd out —
2898                // otherwise it would judge session/continuation eligibility
2899                // by the wrong CLI and could resend a fallback's session id
2900                // to the agent that lost it the seat in the first place.
2901                job.spec = next.clone();
2902                let mut retry = job.clone();
2903                retry.seat = new_seat;
2904                retry.prompt = prompt::implement_in(
2905                    &instruction,
2906                    &job.cwd.to_string_lossy(),
2907                    &language,
2908                    &github_language,
2909                    brief.as_deref(),
2910                    &attachments,
2911                );
2912                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2913                let cache = self.state.config.cache_dir();
2914                let ctx = WaveCtx {
2915                    carry_seats: false,
2916                    pauses: Some((&self.pause, &self.interrupt)),
2917                    run: run_id,
2918                    node: "implement",
2919                    prompts,
2920                    cache: cache.as_deref(),
2921                    round: None,
2922                };
2923                let (fallback_seat, fallback_out) = run_one(
2924                    retry,
2925                    Arc::clone(&self.sem),
2926                    &ctx,
2927                    &mut self.state,
2928                    fallback_attempt,
2929                )
2930                .await;
2931                *seat = fallback_seat;
2932                *out = fallback_out;
2933            }
2934        }
2935        Ok(())
2936    }
2937
2938    /// Ask an implement seat's own CLI to confirm what it started, once, when
2939    /// its reply reported a command whose completion status it never
2940    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
2941    /// that does and does not mean.
2942    ///
2943    /// The completion contract this task asks for, extended to `implement`
2944    /// with the same signal `continue_fix_report` reads for the fixer,
2945    /// rather than a keyword search over the reply or a hard requirement on
2946    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
2947    /// a candidate's CLI turn ended cleanly while a test run it had started
2948    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
2949    /// named in it at all is untouched by this: `commands` is empty, so
2950    /// there is nothing to be unconfirmed.
2951    ///
2952    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
2953    /// this is not about recovering edits that might already be on disk, it
2954    /// is about a result the seat itself never vouched for, which resuming
2955    /// asks for regardless of what the tree already holds. Bounded to one
2956    /// attempt for the same reason `resume_undelivered` is — this is the
2957    /// most expensive node in the graph — and a seat that still cannot
2958    /// confirm on that attempt is left as whatever its (possibly still
2959    /// unconfirmed) reply says; this does not invent a new "failed" reason
2960    /// for a candidate that otherwise produced a real, committed change.
2961    async fn resume_unconfirmed_commands(
2962        &mut self,
2963        results: &mut [(usize, SeatState, AgentOutcome)],
2964        sent: &[SeatJob],
2965        prompts: &Prompts,
2966        run_id: &str,
2967    ) {
2968        for (wi, seat, out) in results.iter_mut() {
2969            let AgentOutcome::Ok(o) = &*out else {
2970                continue;
2971            };
2972            if !has_unconfirmed_command(&o.commands) {
2973                continue;
2974            }
2975            let Some(job) = sent.get(*wi) else { continue };
2976            if !has_context(&job.spec, seat, job.sessions) {
2977                self.state.event(
2978                    "implement",
2979                    format!(
2980                        "{}: the reply named a command whose own CLI never confirmed the exit \
2981                         status of, but there is no session left to resume",
2982                        seat.key
2983                    ),
2984                );
2985                continue;
2986            }
2987            self.state.event(
2988                "implement",
2989                format!(
2990                    "{}: the reply named a command whose own CLI never confirmed the exit \
2991                     status of; resuming the conversation",
2992                    seat.key
2993                ),
2994            );
2995            let mut retry = job.clone();
2996            retry.seat = seat.clone();
2997            retry.prompt = prompt::resume_incomplete(
2998                "a command in your last reply had no confirmed exit status",
2999            );
3000            retry.timeout = retry_budget(job.timeout, true);
3001            retry.stem = format!("{}-confirm", job.stem);
3002            let cache = self.state.config.cache_dir();
3003            let ctx = WaveCtx {
3004                carry_seats: false,
3005                pauses: Some((&self.pause, &self.interrupt)),
3006                run: run_id,
3007                node: "implement",
3008                prompts,
3009                cache: cache.as_deref(),
3010                round: None,
3011            };
3012            let (resumed_seat, resumed) =
3013                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3014            *seat = resumed_seat;
3015            *out = resumed;
3016        }
3017    }
3018
3019    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3020    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3021    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3022    /// that motivated this.
3023    ///
3024    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3025    /// own nudge loop already covers for judge/review/vote seats, and not a
3026    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3027    /// implement seats: here the CLI turn genuinely finished while the node's
3028    /// own work — the fixer's account of what it did — had not. Gated purely
3029    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3030    /// reply, never on any wording in it, so a fixer whose valid, first-try
3031    /// `FixReport` happens to mention having waited on a background test is
3032    /// never resumed — the `Ok(report)` branch at the call site returns
3033    /// before this is ever invoked.
3034    ///
3035    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3036    /// attempt ([`retry_budget`]), nothing attempted once the session is
3037    /// gone, and a quota hit ends the loop immediately rather than retrying a
3038    /// rate limit that fails the same way again.
3039    async fn continue_fix_report(
3040        &mut self,
3041        mut seat: SeatState,
3042        parse_err: String,
3043        job: &SeatJob,
3044        prompts: &Prompts,
3045        run_id: &str,
3046        round: usize,
3047    ) -> (
3048        SeatState,
3049        Option<FixReport>,
3050        Option<String>,
3051        ContinuationRecord,
3052    ) {
3053        let mut last_err = parse_err;
3054        let mut cumulative_wait_ms = 0u64;
3055        let mut attempts = 0usize;
3056        loop {
3057            if !has_context(&job.spec, &seat, job.sessions) {
3058                self.state.event(
3059                    "fix",
3060                    format!(
3061                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3062                         session left to resume into"
3063                    ),
3064                );
3065                let outcome = if attempts == 0 {
3066                    ContinuationOutcome::NoSession
3067                } else {
3068                    ContinuationOutcome::Exhausted
3069                };
3070                return (
3071                    seat,
3072                    None,
3073                    Some(format!("unparsable fix report: {last_err}")),
3074                    ContinuationRecord {
3075                        attempts,
3076                        cumulative_wait_ms,
3077                        outcome,
3078                    },
3079                );
3080            }
3081            if attempts >= MAX_FIX_CONTINUATIONS {
3082                self.state.event(
3083                    "fix",
3084                    format!(
3085                        "round {round}: fixer's reply still had no adoption report after \
3086                         {attempts} continuation(s) ({last_err}); giving up"
3087                    ),
3088                );
3089                return (
3090                    seat,
3091                    None,
3092                    Some(format!(
3093                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3094                    )),
3095                    ContinuationRecord {
3096                        attempts,
3097                        cumulative_wait_ms,
3098                        outcome: ContinuationOutcome::Exhausted,
3099                    },
3100                );
3101            }
3102            attempts += 1;
3103            self.state.event(
3104                "fix",
3105                format!(
3106                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3107                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3108                ),
3109            );
3110            let mut retry = job.clone();
3111            retry.seat = seat.clone();
3112            retry.prompt = prompt::resume_incomplete(&last_err);
3113            retry.timeout = retry_budget(job.timeout, true);
3114            retry.stem = format!("{}-continue{attempts}", job.stem);
3115            let cache = self.state.config.cache_dir();
3116            let ctx = WaveCtx {
3117                carry_seats: false,
3118                pauses: Some((&self.pause, &self.interrupt)),
3119                run: run_id,
3120                node: "fix",
3121                prompts,
3122                cache: cache.as_deref(),
3123                round: Some(round),
3124            };
3125            let (resumed_seat, resumed_out) = run_one(
3126                retry,
3127                Arc::clone(&self.sem),
3128                &ctx,
3129                &mut self.state,
3130                attempts,
3131            )
3132            .await;
3133            seat = resumed_seat;
3134            match resumed_out {
3135                AgentOutcome::Ok(o) => {
3136                    cumulative_wait_ms += o.duration_ms;
3137                    match verdict::extract_json::<FixReport>(&o.text) {
3138                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3139                            self.state.event(
3140                                "fix",
3141                                format!(
3142                                    "round {round}: fixer's adoption report recovered after \
3143                                     {attempts} continuation(s)"
3144                                ),
3145                            );
3146                            return (
3147                                seat,
3148                                Some(report),
3149                                None,
3150                                ContinuationRecord {
3151                                    attempts,
3152                                    cumulative_wait_ms,
3153                                    outcome: ContinuationOutcome::Resumed,
3154                                },
3155                            );
3156                        }
3157                        // The report parsed, but this same reply's own
3158                        // CommandEvidence — the identical record `state.jobs`
3159                        // renders — names a command whose CLI never
3160                        // confirmed an exit status. Read together, that is
3161                        // not a resolved answer: keep nudging rather than
3162                        // accept a report standing next to a command the
3163                        // seat's own CLI cannot vouch for.
3164                        Ok(_) => {
3165                            last_err = "the reply parsed, but it reported a command whose own CLI \
3166                                 never confirmed an exit status"
3167                                .to_owned();
3168                        }
3169                        Err(e) => last_err = e.to_string(),
3170                    }
3171                }
3172                AgentOutcome::Quota(o) => {
3173                    cumulative_wait_ms += o.duration_ms;
3174                    self.state.quota.push(QuotaLoss {
3175                        seat: seat.key.clone(),
3176                        node: "fix".to_owned(),
3177                        at: Timestamp::now(),
3178                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3179                    });
3180                    self.state.event(
3181                        "fix",
3182                        format!(
3183                            "round {round}: continuation rate limited (quota); not retrying now"
3184                        ),
3185                    );
3186                    return (
3187                        seat,
3188                        None,
3189                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3190                        ContinuationRecord {
3191                            attempts,
3192                            cumulative_wait_ms,
3193                            outcome: ContinuationOutcome::QuotaLost,
3194                        },
3195                    );
3196                }
3197                AgentOutcome::Dropped(o) => {
3198                    cumulative_wait_ms += o.duration_ms;
3199                    let why = o
3200                        .dropped
3201                        .as_ref()
3202                        .map(|d| d.why.as_str())
3203                        .unwrap_or("the CLI ended the stream without delivering its answer");
3204                    last_err = format!("the CLI dropped the stream ({why})");
3205                }
3206                AgentOutcome::Failed(e) => last_err = e,
3207                // The continuation was never started; the report stays lost
3208                // and the caller records that, without any handover.
3209                AgentOutcome::ResourceBlocked(e) => {
3210                    return (
3211                        seat,
3212                        None,
3213                        Some(format!("the continuation was not started: {e}")),
3214                        ContinuationRecord {
3215                            attempts: attempts.saturating_sub(1),
3216                            cumulative_wait_ms,
3217                            outcome: ContinuationOutcome::NoSession,
3218                        },
3219                    );
3220                }
3221            }
3222        }
3223    }
3224
3225    fn after_implement(&mut self) -> Result<()> {
3226        if !self.state.candidate_preservation.is_empty() {
3227            bail!("candidate preservation incomplete; judging is unsafe");
3228        }
3229        // Scan every candidate patch once the set is complete.
3230        if self.state.leaks.is_empty() {
3231            let cfg = self.state.config.blind.clone();
3232            let mut leaks = Vec::new();
3233            for c in &self.state.candidates {
3234                let Some(patch) =
3235                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3236                else {
3237                    continue;
3238                };
3239                leaks.extend(blind::scan(
3240                    &format!("candidate {} patch", c.label),
3241                    &patch,
3242                    &cfg.vendor_tokens,
3243                ));
3244            }
3245            if !leaks.is_empty() {
3246                let summary = leaks
3247                    .iter()
3248                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3249                    .collect::<Vec<_>>()
3250                    .join(", ");
3251                match cfg.on_leak {
3252                    LeakPolicy::Fail => {
3253                        self.state.status = RunStatus::Failed;
3254                        self.state
3255                            .event("blind", format!("vendor text in a patch: {summary}"));
3256                        self.state.leaks = leaks;
3257                        self.state.save()?;
3258                        self.settle_questions();
3259                        bail!(
3260                            "blind.on_leak = \"fail\" and vendor text reached a \
3261                             judged patch: {summary}"
3262                        );
3263                    }
3264                    LeakPolicy::Redact => self.state.event(
3265                        "blind",
3266                        format!("redacting vendor text for judging: {summary}"),
3267                    ),
3268                    LeakPolicy::Warn => self.state.event(
3269                        "blind",
3270                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3271                    ),
3272                }
3273                self.state.leaks = leaks;
3274            }
3275        }
3276
3277        if self.state.viable().is_empty() {
3278            if self.state.all_candidates_verified_noop() {
3279                // Every candidate agreed, with evidence the adoption guard
3280                // accepted, that nothing belongs in this worktree. That is
3281                // not the same fact as a candidate that simply failed to
3282                // write anything, and settling it as an ordinary `Failed`
3283                // (see `SCHEMA`'s doc for schema 10) is what let two of
3284                // task 391f's attempts burn a retry each re-discovering the
3285                // same already-landed fix. Terminal either way, so `judge`
3286                // must never run over an empty candidate set — unlike the
3287                // `Failed` branch below this returns `Ok`, not an error:
3288                // nothing here failed.
3289                self.state.status = RunStatus::VerifiedNoop;
3290                self.state.save()?;
3291                self.settle_questions();
3292                return Ok(());
3293            }
3294            self.state.status = RunStatus::Failed;
3295            self.state.save()?;
3296            self.settle_questions();
3297            bail!("no candidate produced a change; nothing to judge");
3298        }
3299        self.state.status = RunStatus::Judging;
3300        self.state.save()?;
3301        Ok(())
3302    }
3303
3304    // --------------------------------------------------------------- judge
3305
3306    async fn judge(&mut self) -> Result<()> {
3307        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3308        // agent files with `magi task add` name the run that paid for it. The
3309        // prompt overlay is cloned alongside it because the waves borrow it
3310        // while `self` is mutably borrowed by the node's own bookkeeping.
3311        let run_id = self.state.id.clone();
3312        let prompts = self.state.config.prompts.clone();
3313        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3314            return Ok(());
3315        }
3316        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3317        if viable.len() == 1 {
3318            // Recorded so this is a one-time event: `judgements` stays empty
3319            // either way, which without this flag is indistinguishable from
3320            // "not yet judged" on the next reentry — and status is left
3321            // untouched, so a later node's conclusion (e.g. `Blocked` after
3322            // the review budget ran out) survives a resume instead of being
3323            // clobbered back to `Judging` by this node running again.
3324            self.state.judge_skipped = true;
3325            self.state.event(
3326                "judge",
3327                format!(
3328                    "only candidate {} produced a change; judging skipped",
3329                    viable[0].label
3330                ),
3331            );
3332            self.state.save()?;
3333            return Ok(());
3334        }
3335        self.state.status = RunStatus::Judging;
3336
3337        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3338        let language = self.state.config.graph.language.clone();
3339        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3340        let sessions = self.state.config.graph.sessions;
3341        let artifacts = agent::artifacts_dir(&self.state.dir());
3342        let root = self.state.worktree_root();
3343        let base_short = short(&self.state.base_commit);
3344
3345        let mut jobs = Vec::new();
3346        let mut orders = Vec::new();
3347        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3348            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3349            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3350            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3351            let seat_key = format!("judge-{}", j + 1);
3352            let seat = self.seat(&seat_key, &spec.id);
3353            jobs.push(SeatJob {
3354                prompt: prompt::judge(
3355                    &self.state.instruction,
3356                    &views,
3357                    self.roles.judges.len(),
3358                    &base_short,
3359                    &language,
3360                ),
3361                spec,
3362                seat,
3363                cwd: root.join(format!("judge-{}", j + 1)),
3364                timeout,
3365                allow_write: false,
3366                sessions,
3367                artifacts: artifacts.clone(),
3368                stem: format!("judge-{}", j + 1),
3369                handover: None,
3370            });
3371        }
3372
3373        self.state.event(
3374            "judge",
3375            format!(
3376                "{} judges ranking {} candidates blind",
3377                jobs.len(),
3378                viable.len()
3379            ),
3380        );
3381        let labels_for_check = labels.clone();
3382        let mut quota_losses = Vec::new();
3383        let cache = self.state.config.cache_dir();
3384        let ctx = WaveCtx {
3385            carry_seats: false,
3386            pauses: Some((&self.pause, &self.interrupt)),
3387            run: &run_id,
3388            node: "judge",
3389            prompts: &prompts,
3390            cache: cache.as_deref(),
3391            round: None,
3392        };
3393        let results = ask_json_wave::<Ranking>(
3394            jobs,
3395            Arc::clone(&self.sem),
3396            self.state.config.graph.retries,
3397            &self.roles.judge_roster,
3398            &ctx,
3399            &mut quota_losses,
3400            &mut self.state,
3401            &move |r: &Ranking| r.validate(&labels_for_check),
3402        )
3403        .await;
3404        if self.state.resource_block.is_some() {
3405            return self.stop_for_resource();
3406        }
3407        self.state.quota.extend(quota_losses);
3408
3409        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3410            let agent_id = seat.agent.clone();
3411            self.state.seats.insert(seat.key.clone(), seat);
3412            let mut record = Judgement {
3413                judge: j + 1,
3414                seat: format!("judge-{}", j + 1),
3415                agent: agent_id,
3416                ranking: Vec::new(),
3417                reasons: BTreeMap::new(),
3418                confidence: None,
3419                strengths: BTreeMap::new(),
3420                order: orders[j].clone(),
3421                failed: None,
3422                duration_ms: 0,
3423            };
3424            match res {
3425                Ok((ranking, out)) => {
3426                    record.ranking = ranking.normalized();
3427                    record.reasons = ranking.reasons;
3428                    record.strengths = ranking.strengths;
3429                    record.confidence = ranking.confidence;
3430                    record.duration_ms = out.duration_ms;
3431                    self.state.event(
3432                        "judge",
3433                        format!(
3434                            "judge {} ranked {}",
3435                            j + 1,
3436                            record.ranking.iter().collect::<String>()
3437                        ),
3438                    );
3439                }
3440                Err(e) => {
3441                    record.failed = Some(e.to_string());
3442                    self.state
3443                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3444                }
3445            }
3446            self.state.judgements.push(record);
3447            self.state.save()?;
3448        }
3449        Ok(())
3450    }
3451
3452    // ---------------------------------------------------------- deliberate
3453
3454    async fn deliberate(&mut self) -> Result<()> {
3455        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3456        // agent files with `magi task add` name the run that paid for it. The
3457        // prompt overlay is cloned alongside it because the waves borrow it
3458        // while `self` is mutably borrowed by the node's own bookkeeping.
3459        let run_id = self.state.id.clone();
3460        let prompts = self.state.config.prompts.clone();
3461        if !self.state.deliberation.is_empty() {
3462            return Ok(());
3463        }
3464        let tops: Vec<char> = self
3465            .state
3466            .judgements
3467            .iter()
3468            .filter_map(|j| j.ranking.first().copied())
3469            .collect();
3470        let rounds = self.state.config.graph.deliberate_rounds;
3471        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3472            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3473                self.state.event(
3474                    "deliberate",
3475                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3476                );
3477            }
3478            self.state.status = RunStatus::Voting;
3479            self.state.save()?;
3480            return Ok(());
3481        }
3482
3483        self.state.status = RunStatus::Deliberating;
3484        self.state.event(
3485            "deliberate",
3486            format!(
3487                "split: first choices were {} — opening {rounds} round(s)",
3488                tops.iter().collect::<String>()
3489            ),
3490        );
3491
3492        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3493        let language = self.state.config.graph.language.clone();
3494        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3495        let sessions = self.state.config.graph.sessions;
3496        let artifacts = agent::artifacts_dir(&self.state.dir());
3497        let root = self.state.worktree_root();
3498        let base_short = short(&self.state.base_commit);
3499
3500        // Judges argue in sequence so that a turn can answer the one before it;
3501        // that is the difference between deliberation and three parallel
3502        // monologues.
3503        for round in 1..=rounds {
3504            let mut turns: Vec<DeliberationTurn> = Vec::new();
3505            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3506                if self.state.judgements[j].failed.is_some() {
3507                    continue;
3508                }
3509                let seat_key = format!("judge-{}", j + 1);
3510                let spec = self.occupant(&seat_key, spec);
3511                let mut seat = self.seat(&seat_key, &spec.id);
3512                let transcript = self.transcript(&turns, j);
3513                let build = |context: Option<&str>| {
3514                    prompt::deliberate(
3515                        &self.state.instruction,
3516                        context,
3517                        &transcript,
3518                        round,
3519                        rounds,
3520                        &language,
3521                    )
3522                };
3523                let block = self.candidate_block(&viable, &base_short);
3524                let full = build(Some(&block));
3525                let text = if has_context(&spec, &seat, sessions) {
3526                    build(None)
3527                } else {
3528                    full.clone()
3529                };
3530                let job = SeatJob {
3531                    spec,
3532                    seat: seat.clone(),
3533                    prompt: text,
3534                    cwd: root.join(format!("judge-{}", j + 1)),
3535                    timeout,
3536                    allow_write: false,
3537                    sessions,
3538                    artifacts: artifacts.clone(),
3539                    stem: format!("delib-{round}-judge-{}", j + 1),
3540                    handover: Some(full),
3541                };
3542                let cache = self.state.config.cache_dir();
3543                let ctx = WaveCtx {
3544                    carry_seats: false,
3545                    pauses: Some((&self.pause, &self.interrupt)),
3546                    run: &run_id,
3547                    node: "deliberate",
3548                    prompts: &prompts,
3549                    cache: cache.as_deref(),
3550                    round: None,
3551                };
3552                // A turn is never nudged (`retries` 0): a failed seat is
3553                // handed to the next roster agent, which gets the full
3554                // context. An empty answer is a turn, not a failure.
3555                let mut losses = Vec::new();
3556                let mut results = ask_wave_with::<String>(
3557                    vec![job],
3558                    Arc::clone(&self.sem),
3559                    0,
3560                    &self.roles.judge_roster,
3561                    &ctx,
3562                    &mut losses,
3563                    &mut self.state,
3564                    &|text: &str| {
3565                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3566                    },
3567                )
3568                .await;
3569                self.state.quota.extend(losses);
3570                let (updated, res, _) = results.pop().expect("one job in, one result out");
3571                seat = updated;
3572                let agent_id = seat.agent.clone();
3573                self.state.seats.insert(seat.key.clone(), seat);
3574                let body = match res {
3575                    Ok((body, _)) => body,
3576                    // Skip the seat; a CLI's raw error JSON is never read as
3577                    // this judge's position.
3578                    Err(e) => {
3579                        self.state
3580                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3581                        continue;
3582                    }
3583                };
3584                let tentative = verdict::extract_json::<Position>(&body)
3585                    .ok()
3586                    .and_then(|p| p.tentative)
3587                    .and_then(|s| s.trim().chars().next())
3588                    .map(|c| c.to_ascii_uppercase());
3589                self.state.event(
3590                    "deliberate",
3591                    format!(
3592                        "round {round}: judge {} now favours {}",
3593                        j + 1,
3594                        tentative.map_or("—".to_owned(), |c| c.to_string())
3595                    ),
3596                );
3597                turns.push(DeliberationTurn {
3598                    judge: j + 1,
3599                    agent: agent_id,
3600                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3601                    tentative,
3602                });
3603            }
3604            self.state
3605                .deliberation
3606                .push(DeliberationRound { round, turns });
3607            self.state.save()?;
3608        }
3609
3610        self.state.status = RunStatus::Voting;
3611        self.state.save()?;
3612        Ok(())
3613    }
3614
3615    // ---------------------------------------------------------------- vote
3616
3617    async fn vote(&mut self) -> Result<()> {
3618        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3619        // agent files with `magi task add` name the run that paid for it. The
3620        // prompt overlay is cloned alongside it because the waves borrow it
3621        // while `self` is mutably borrowed by the node's own bookkeeping.
3622        let run_id = self.state.id.clone();
3623        let prompts = self.state.config.prompts.clone();
3624        if !self.state.votes.is_empty() {
3625            return Ok(());
3626        }
3627        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3628        if viable.len() == 1 {
3629            return Ok(());
3630        }
3631        self.state.status = RunStatus::Voting;
3632
3633        let language = self.state.config.graph.language.clone();
3634        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3635        let sessions = self.state.config.graph.sessions;
3636        let artifacts = agent::artifacts_dir(&self.state.dir());
3637        let root = self.state.worktree_root();
3638        let base_short = short(&self.state.base_commit);
3639        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3640
3641        let mut jobs = Vec::new();
3642        let mut seats_at = Vec::new();
3643        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3644            if self
3645                .state
3646                .judgements
3647                .get(j)
3648                .is_some_and(|r| r.failed.is_some())
3649            {
3650                continue;
3651            }
3652            let seat_key = format!("judge-{}", j + 1);
3653            let spec = self.occupant(&seat_key, spec);
3654            let seat = self.seat(&seat_key, &spec.id);
3655            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3656            let text = if has_context(&spec, &seat, sessions) {
3657                prompt::final_vote(&viable, &language)
3658            } else {
3659                full.clone()
3660            };
3661            jobs.push(SeatJob {
3662                spec,
3663                seat,
3664                prompt: text,
3665                cwd: root.join(format!("judge-{}", j + 1)),
3666                timeout,
3667                allow_write: false,
3668                sessions,
3669                artifacts: artifacts.clone(),
3670                stem: format!("vote-judge-{}", j + 1),
3671                handover: Some(full),
3672            });
3673            seats_at.push(j);
3674        }
3675
3676        self.state.event(
3677            "vote",
3678            format!(
3679                "collecting {} final votes one by one, privately",
3680                jobs.len()
3681            ),
3682        );
3683        let allowed = viable.clone();
3684        let mut quota_losses = Vec::new();
3685        let cache = self.state.config.cache_dir();
3686        let ctx = WaveCtx {
3687            carry_seats: false,
3688            pauses: Some((&self.pause, &self.interrupt)),
3689            run: &run_id,
3690            node: "vote",
3691            prompts: &prompts,
3692            cache: cache.as_deref(),
3693            round: None,
3694        };
3695        let results = ask_json_wave::<FinalVote>(
3696            jobs,
3697            Arc::clone(&self.sem),
3698            self.state.config.graph.retries,
3699            &self.roles.judge_roster,
3700            &ctx,
3701            &mut quota_losses,
3702            &mut self.state,
3703            &move |v: &FinalVote| match v.label() {
3704                Some(c) if allowed.contains(&c) => Ok(()),
3705                other => bail!("vote {other:?} is not one of {allowed:?}"),
3706            },
3707        )
3708        .await;
3709        if self.state.resource_block.is_some() {
3710            return self.stop_for_resource();
3711        }
3712        self.state.quota.extend(quota_losses);
3713
3714        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3715            let agent_id = seat.agent.clone();
3716            self.state.seats.insert(seat.key.clone(), seat);
3717            let initial = self
3718                .state
3719                .judgements
3720                .get(j)
3721                .and_then(|r| r.ranking.first().copied());
3722            let mut record = VoteRecord {
3723                judge: j + 1,
3724                agent: agent_id,
3725                vote: None,
3726                reason: String::new(),
3727                changed: false,
3728            };
3729            match res {
3730                Ok((v, _)) => {
3731                    record.vote = v.label();
3732                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3733                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3734                    self.state.event(
3735                        "vote",
3736                        format!(
3737                            "judge {} voted {}{}",
3738                            j + 1,
3739                            record.vote.unwrap_or('?'),
3740                            if record.changed { " (changed)" } else { "" }
3741                        ),
3742                    );
3743                }
3744                Err(e) => {
3745                    self.state
3746                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3747                }
3748            }
3749            self.state.votes.push(record);
3750            self.state.save()?;
3751        }
3752        Ok(())
3753    }
3754
3755    // --------------------------------------------------------------- tally
3756
3757    fn tally(&mut self) -> Result<()> {
3758        if self.state.tally.is_some() {
3759            return Ok(());
3760        }
3761        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3762        let tops: Vec<char> = self
3763            .state
3764            .judgements
3765            .iter()
3766            .filter_map(|j| j.ranking.first().copied())
3767            .collect();
3768        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3769
3770        // A judge whose private vote failed still counted once, in the initial
3771        // ranking; using it beats discarding a whole seat.
3772        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3773        let mut cast: Vec<char> = Vec::new();
3774        for (i, j) in self.state.judgements.iter().enumerate() {
3775            let vote = self
3776                .state
3777                .votes
3778                .iter()
3779                .find(|v| v.judge == i + 1)
3780                .and_then(|v| v.vote)
3781                .or_else(|| j.ranking.first().copied());
3782            if let Some(v) = vote {
3783                *first_choice.entry(v).or_insert(0) += 1;
3784                cast.push(v);
3785            }
3786        }
3787
3788        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3789        for j in &self.state.judgements {
3790            let n = j.ranking.len();
3791            for (pos, label) in j.ranking.iter().enumerate() {
3792                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3793            }
3794        }
3795
3796        let best = first_choice.values().copied().max().unwrap_or(0);
3797        let mut leaders: Vec<char> = first_choice
3798            .iter()
3799            .filter(|(_, v)| **v == best)
3800            .map(|(k, _)| *k)
3801            .collect();
3802        let mut tie_break = None;
3803        if leaders.len() > 1 {
3804            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3805            let borda_leaders: Vec<char> = leaders
3806                .iter()
3807                .copied()
3808                .filter(|l| borda[l] == top_borda)
3809                .collect();
3810            tie_break = Some(if borda_leaders.len() == 1 {
3811                format!(
3812                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3813                    leaders.len()
3814                )
3815            } else {
3816                format!(
3817                    "{} way tie on both first-choice votes and Borda points, broken by label order",
3818                    leaders.len()
3819                )
3820            });
3821            leaders = borda_leaders;
3822            leaders.sort_unstable();
3823        }
3824        let winner = *leaders
3825            .first()
3826            .or(viable.first())
3827            .context("no candidate to declare a winner from")?;
3828
3829        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3830        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3831        let deliberated = !self.state.deliberation.is_empty();
3832
3833        // Whose verdict is this? A rate-limited seat is absent even if it
3834        // ranked before the limit hit, so presence is measured against the
3835        // recorded losses, not just "did a ranking ever appear".
3836        let quota_seats: std::collections::BTreeSet<&str> =
3837            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3838        let mut present = 0usize;
3839        for (i, j) in self.state.judgements.iter().enumerate() {
3840            if quota_seats.contains(j.seat.as_str()) {
3841                continue;
3842            }
3843            let ranked = !j.ranking.is_empty() && j.failed.is_none();
3844            let voted = self
3845                .state
3846                .votes
3847                .iter()
3848                .any(|v| v.judge == i + 1 && v.vote.is_some());
3849            if ranked || voted {
3850                present += 1;
3851            }
3852        }
3853        // Strict majority of the configured panel. A bare majority is real
3854        // signal we can act on, while a minority verdict must never stand in
3855        // for a healthy one. A one-candidate run needs no panel at all, and
3856        // `judges` stays `0` rather than the roster size a panel that never
3857        // sat would otherwise be credited with.
3858        let needs_quorum = viable.len() > 1;
3859        let judges_total = if needs_quorum {
3860            self.roles.judges.len()
3861        } else {
3862            0
3863        };
3864        let quorum = if needs_quorum {
3865            judges_total / 2 + 1
3866        } else {
3867            0
3868        };
3869        let met_quorum = !needs_quorum || present >= quorum;
3870        let uncontested = (!needs_quorum).then(|| {
3871            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3872        });
3873
3874        self.state.event(
3875            "tally",
3876            match &uncontested {
3877                Some(reason) => format!("winner {winner} — {reason}"),
3878                None => format!(
3879                    "winner {winner} — votes {} | initial {} | {} changed | \
3880                     {present}/{judges_total} judges{}",
3881                    first_choice
3882                        .iter()
3883                        .map(|(k, v)| format!("{k}:{v}"))
3884                        .collect::<Vec<_>>()
3885                        .join(" "),
3886                    if unanimous_initial {
3887                        "unanimous"
3888                    } else {
3889                        "split"
3890                    },
3891                    changed_votes,
3892                    if met_quorum {
3893                        String::new()
3894                    } else {
3895                        format!(" — below quorum ({quorum} required)")
3896                    },
3897                ),
3898            },
3899        );
3900        if !met_quorum {
3901            self.state.event(
3902                "stall",
3903                format!(
3904                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3905                     the run stops here, resumable"
3906                ),
3907            );
3908        }
3909        self.state.tally = Some(Tally {
3910            first_choice,
3911            borda,
3912            winner,
3913            rankings: tops.len(),
3914            unanimous_initial,
3915            deliberated,
3916            changed_votes,
3917            unanimous_final,
3918            tie_break,
3919            judges: judges_total,
3920            present,
3921            quorum,
3922            met_quorum,
3923            uncontested,
3924        });
3925        self.state.status = if met_quorum {
3926            RunStatus::Reviewing
3927        } else {
3928            RunStatus::Stalled
3929        };
3930        self.state.save()?;
3931        Ok(())
3932    }
3933
3934    // ------------------------------------------------------------- recover
3935
3936    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
3937    /// resumed toward completion once the transient cause clears.
3938    ///
3939    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
3940    /// it toward the quorum, which is exactly the set of seats whose absence
3941    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
3942    /// not depend on which node happened to hit the limit), or an ordinary
3943    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
3944    /// seat is never disturbed.
3945    ///
3946    /// A seat that now answers with a usable ranking is "recovered": its
3947    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
3948    /// `tally` counts it present again), and its vote re-collected. A seat that
3949    /// still fails keeps its loss and stays absent.
3950    ///
3951    /// Returns `true` when the re-tally restores the quorum (the run may proceed
3952    /// to review/gate/merge), `false` when it is still below quorum (the run
3953    /// stays `Stalled`, still resumable for a later retry).
3954    #[allow(clippy::too_many_lines)]
3955    async fn recover_stall(&mut self) -> Result<bool> {
3956        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3957        // agent files with `magi task add` name the run that paid for it. The
3958        // prompt overlay is cloned alongside it because the waves borrow it
3959        // while `self` is mutably borrowed by the node's own bookkeeping.
3960        let run_id = self.state.id.clone();
3961        let prompts = self.state.config.prompts.clone();
3962        // Absent seats = quota-lost at any node, or failed outright. Mirroring
3963        // `tally`'s presence test (rather than the old quota-judge/vote filter)
3964        // is what keeps a non-quota collapse — or a quota loss recorded at the
3965        // deliberate node — from being a permanent dead-end on `--resume`.
3966        let quota_seats: BTreeSet<&str> =
3967            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3968        let absent: Vec<String> = self
3969            .state
3970            .judgements
3971            .iter()
3972            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3973            .map(|j| j.seat.clone())
3974            .collect();
3975        if absent.is_empty() {
3976            return Ok(false);
3977        }
3978        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3979        if viable.len() <= 1 {
3980            return Ok(false);
3981        }
3982        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3983        let language = self.state.config.graph.language.clone();
3984        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3985        let sessions = self.state.config.graph.sessions;
3986        let artifacts = agent::artifacts_dir(&self.state.dir());
3987        let root = self.state.worktree_root();
3988        let base_short = short(&self.state.base_commit);
3989        let candidates: Vec<Candidate> = viable.clone();
3990
3991        // Map each absent seat key to its 0-based position in `roles.judges`.
3992        let mut positions: Vec<usize> = absent
3993            .iter()
3994            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3995            .collect();
3996        if positions.is_empty() {
3997            return Ok(false);
3998        }
3999        positions.sort_unstable();
4000        positions.dedup();
4001
4002        // Re-rank the lost seats, one blind prompt each.
4003        let mut judge_jobs = Vec::new();
4004        for &j in &positions {
4005            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4006            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4007            let seat_key = format!("judge-{}", j + 1);
4008            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4009            let seat = self.seat(&seat_key, &spec.id);
4010            judge_jobs.push(SeatJob {
4011                spec,
4012                seat,
4013                prompt: prompt::judge(
4014                    &self.state.instruction,
4015                    &views,
4016                    self.roles.judges.len(),
4017                    &base_short,
4018                    &language,
4019                ),
4020                cwd: root.join(seat_key),
4021                timeout,
4022                allow_write: false,
4023                sessions,
4024                artifacts: artifacts.clone(),
4025                stem: format!("judge-{}-recover", j + 1),
4026                handover: None,
4027            });
4028        }
4029
4030        let labels_for_check = labels.clone();
4031        let mut judge_losses = Vec::new();
4032        let retries = self.state.config.graph.retries;
4033        let cache = self.state.config.cache_dir();
4034        let ctx = WaveCtx {
4035            carry_seats: false,
4036            pauses: Some((&self.pause, &self.interrupt)),
4037            run: &run_id,
4038            node: "judge",
4039            prompts: &prompts,
4040            cache: cache.as_deref(),
4041            round: None,
4042        };
4043        let results = ask_json_wave::<Ranking>(
4044            judge_jobs,
4045            Arc::clone(&self.sem),
4046            retries,
4047            &self.roles.judge_roster,
4048            &ctx,
4049            &mut judge_losses,
4050            &mut self.state,
4051            &move |r: &Ranking| r.validate(&labels_for_check),
4052        )
4053        .await;
4054        if self.state.resource_block.is_some() {
4055            self.stop_for_resource()?;
4056            return Ok(false);
4057        }
4058
4059        // Refresh the judgement of every seat that ranked again.
4060        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4061        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4062            let agent_id = seat.agent.clone();
4063            self.state.seats.insert(seat.key.clone(), seat);
4064            let record = &mut self.state.judgements[j];
4065            match res {
4066                Ok((ranking, out)) => {
4067                    record.agent = agent_id;
4068                    record.ranking = ranking.normalized();
4069                    record.reasons = ranking.reasons;
4070                    record.strengths = ranking.strengths;
4071                    record.confidence = ranking.confidence;
4072                    record.failed = None;
4073                    record.duration_ms = out.duration_ms;
4074                    recovered.insert(j);
4075                    self.state.event(
4076                        "recover",
4077                        format!("judge {} ranked again after the limit", j + 1),
4078                    );
4079                }
4080                Err(e) => {
4081                    self.state
4082                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4083                }
4084            }
4085        }
4086
4087        // Re-ask the votes of the seats that recovered a ranking.
4088        let mut vote_jobs = Vec::new();
4089        let mut vote_pos: Vec<usize> = Vec::new();
4090        for &j in &recovered {
4091            let seat_key = format!("judge-{}", j + 1);
4092            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4093            let seat = self.seat(&seat_key, &spec.id);
4094            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4095            let text = if has_context(&spec, &seat, sessions) {
4096                prompt::final_vote(&labels, &language)
4097            } else {
4098                full.clone()
4099            };
4100            vote_jobs.push(SeatJob {
4101                spec,
4102                seat,
4103                prompt: text,
4104                cwd: root.join(seat_key),
4105                timeout,
4106                allow_write: false,
4107                sessions,
4108                artifacts: artifacts.clone(),
4109                stem: format!("vote-judge-{}-recover", j + 1),
4110                handover: Some(full),
4111            });
4112            vote_pos.push(j);
4113        }
4114        let allowed = labels.clone();
4115        let mut vote_losses = Vec::new();
4116        let vote_retries = self.state.config.graph.retries;
4117        let vote_cache = self.state.config.cache_dir();
4118        let ctx = WaveCtx {
4119            carry_seats: false,
4120            pauses: Some((&self.pause, &self.interrupt)),
4121            run: &run_id,
4122            node: "vote",
4123            prompts: &prompts,
4124            cache: vote_cache.as_deref(),
4125            round: None,
4126        };
4127        let votes = ask_json_wave::<FinalVote>(
4128            vote_jobs,
4129            Arc::clone(&self.sem),
4130            vote_retries,
4131            &self.roles.judge_roster,
4132            &ctx,
4133            &mut vote_losses,
4134            &mut self.state,
4135            &move |v: &FinalVote| match v.label() {
4136                Some(c) if allowed.contains(&c) => Ok(()),
4137                other => bail!("vote {other:?} is not one of {allowed:?}"),
4138            },
4139        )
4140        .await;
4141        if self.state.resource_block.is_some() {
4142            self.stop_for_resource()?;
4143            return Ok(false);
4144        }
4145        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4146            let agent_id = seat.agent.clone();
4147            self.state.seats.insert(seat.key.clone(), seat);
4148            match res {
4149                Ok((v, _)) => {
4150                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4151                        rec.vote = v.label();
4152                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4153                    } else {
4154                        self.state.votes.push(VoteRecord {
4155                            judge: j + 1,
4156                            agent: agent_id,
4157                            vote: v.label(),
4158                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4159                            changed: false,
4160                        });
4161                    }
4162                    self.state.event(
4163                        "recover",
4164                        format!("judge {} voted again after the limit", j + 1),
4165                    );
4166                }
4167                Err(e) => {
4168                    self.state
4169                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4170                }
4171            }
4172        }
4173
4174        // A seat that ranked again is present even if its re-vote failed —
4175        // `tally` falls back to the initial ranking's first choice — so clear
4176        // its quota loss. Seats that still fail keep theirs and stay absent.
4177        let recovered_keys: BTreeSet<String> = recovered
4178            .iter()
4179            .map(|&j| format!("judge-{}", j + 1))
4180            .collect();
4181        self.state
4182            .quota
4183            .retain(|q| !recovered_keys.contains(&q.seat));
4184        // A seat that hit the limit again is a fresh loss, not the old one:
4185        // replace the stale entry so the history stays one-per-seat and the
4186        // daemon can tell this attempt's loss from a previous session's.
4187        for loss in judge_losses.into_iter().chain(vote_losses) {
4188            if recovered_keys.contains(&loss.seat) {
4189                continue;
4190            }
4191            self.state.quota.retain(|q| q.seat != loss.seat);
4192            self.state.quota.push(loss);
4193        }
4194
4195        // Recompute the verdict from the refreshed panel.
4196        self.state.tally = None;
4197        self.tally()?;
4198        Ok(self
4199            .state
4200            .tally
4201            .as_ref()
4202            .map(|t| t.met_quorum)
4203            .unwrap_or(false))
4204    }
4205
4206    // ---------------------------------------------------------------- graft
4207
4208    /// Port the strengths the judges attributed to the losing candidates onto
4209    /// the winner, once, between the tally and the review loop.
4210    ///
4211    /// Best-effort by construction: it returns nothing and fails nothing. A
4212    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4213    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4214    /// the run carries on as if the phase were off. See [`graft`] for the
4215    /// decisions and `AGENTS.md` for the invariants.
4216    ///
4217    /// The one exception is a rollback that could not be confirmed: the
4218    /// rejected commit may still be the winner's tip, so the run is blocked
4219    /// (resumable; the record stays `Running`, which makes a resume retry the
4220    /// restore) instead of reviewing, gating and merging it.
4221    async fn graft(&mut self) -> Result<()> {
4222        let Err(e) = self.graft_inner().await else {
4223            return Ok(());
4224        };
4225        if e.is::<crate::repo_lock::ResourceBlocked>() {
4226            return Err(e);
4227        }
4228        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4229            self.state.status = RunStatus::Blocked;
4230            self.state.event("graft", format!("blocked: {e:#}"));
4231            self.state.save()?;
4232            return Err(e);
4233        }
4234        // An error after the `Running` record was saved may have left the
4235        // fixer's work in the tree: the record must not stay unsettled
4236        // (a later resume would reset past the review fixes), so confirm the
4237        // restore now, or block.
4238        if let Some(rec) = self.state.graft.clone()
4239            && !rec.outcome.settled()
4240        {
4241            if let Some(winner) = self.state.winner().cloned() {
4242                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4243                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4244                        return Err(re);
4245                    }
4246                    self.state.status = RunStatus::Blocked;
4247                    self.state.event("graft", format!("blocked: {re:#}"));
4248                    self.state.save()?;
4249                    return Err(re);
4250                }
4251            }
4252            if let Some(g) = self.state.graft.as_mut() {
4253                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4254            }
4255        }
4256        self.state.event("graft", format!("skipped: {e:#}"));
4257        if let Err(e) = self.state.save() {
4258            tracing::warn!("save after graft: {e:#}");
4259        }
4260        Ok(())
4261    }
4262
4263    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4264    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4265    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4266        let _coordination = self.coordinate(wt).await?;
4267        crate::recovery::restore_repair(wt, pre)
4268            .await
4269            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4270    }
4271
4272    /// A skip is a run event, recorded once however often the node is re-entered.
4273    fn graft_skipped(&mut self, why: &str) {
4274        let message = format!("skipped: {why}");
4275        if !self
4276            .state
4277            .events
4278            .iter()
4279            .any(|e| e.node == "graft" && e.message == message)
4280        {
4281            self.state.event("graft", message);
4282        }
4283    }
4284
4285    async fn graft_inner(&mut self) -> Result<()> {
4286        // A round that was started and never finished (crash, kill): the
4287        // fixer may have left anything in the tree, so put the winner back.
4288        if let Some(rec) = self.state.graft.clone()
4289            && !rec.outcome.settled()
4290        {
4291            // Reviews exist only if the graft ended without its record being
4292            // settled (a swallowed error): the tree has moved on since, and
4293            // a reset to `pre` would discard the review fixes.
4294            if self.state.reviews.is_empty()
4295                && let Some(winner) = self.state.winner().cloned()
4296            {
4297                self.graft_restore(&winner.worktree, &rec.pre).await?;
4298            }
4299            if rec.outcome == GraftOutcome::WaitingForRestore {
4300                self.state.graft = None;
4301                self.state.save()?;
4302            } else {
4303                if let Some(g) = self.state.graft.as_mut() {
4304                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4305                }
4306                self.state.event(
4307                    "graft",
4308                    "rolled back: the previous round was interrupted; the winner is untouched",
4309                );
4310                self.state.save()?;
4311                return Ok(());
4312            }
4313        }
4314        let plan = match graft::plan(&self.state) {
4315            Ok(plan) => plan,
4316            Err(skip) => {
4317                if skip != graft::Skip::AlreadyDone {
4318                    self.graft_skipped(skip.reason());
4319                    self.state.save()?;
4320                }
4321                return Ok(());
4322            }
4323        };
4324        let Some(winner) = self.state.winner().cloned() else {
4325            return Ok(());
4326        };
4327        let wt = winner.worktree.clone();
4328        if !git::is_clean(&wt).await? {
4329            self.graft_skipped("the winner's worktree has uncommitted changes");
4330            self.state.save()?;
4331            return Ok(());
4332        }
4333        let pre = git::rev_parse(&wt, "HEAD").await?;
4334        // Saved before the fixer is called: a stop from here on is repaired
4335        // by the branch above on the next entry.
4336        self.state.graft = Some(GraftRecord {
4337            pre: pre.clone(),
4338            post: None,
4339            lines: None,
4340            limit: None,
4341            agent: None,
4342            outcome: GraftOutcome::Running,
4343            requested: graft::requested(&plan),
4344            adoption: None,
4345            adoption_error: None,
4346        });
4347        self.state.event(
4348            "graft",
4349            format!(
4350                "porting strengths of candidate(s) {} onto {}",
4351                plan.strengths
4352                    .iter()
4353                    .map(|(l, _, _)| l.to_string())
4354                    .collect::<Vec<_>>()
4355                    .join(", "),
4356                plan.winner
4357            ),
4358        );
4359        self.state.save()?;
4360
4361        let fix_prompt = prompt::graft(
4362            &self.state.instruction,
4363            plan.winner,
4364            &plan.strengths,
4365            &self.state.config.graph.language,
4366        );
4367        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4368        let sessions = self.state.config.graph.sessions;
4369        let artifacts = agent::artifacts_dir(&self.state.dir());
4370        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4371        let (_, seat, out) = self
4372            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4373                SeatJob {
4374                    prompt: fix_prompt.clone(),
4375                    spec,
4376                    seat,
4377                    cwd: wt.clone(),
4378                    timeout,
4379                    allow_write: true,
4380                    sessions,
4381                    artifacts: artifacts.clone(),
4382                    stem: "graft".to_owned(),
4383                    handover: None,
4384                }
4385            })
4386            .await;
4387        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4388            // Preserve the need to restore earlier fallback edits without
4389            // confusing an unstarted seat with an interrupted invocation.
4390            self.state.graft.as_mut().expect("graft record").outcome =
4391                GraftOutcome::WaitingForRestore;
4392            return self.stop_for_resource();
4393        }
4394        let mut why: Option<String> = None;
4395        match &out {
4396            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4397                why = Some("the fixer produced nothing usable".to_owned());
4398            }
4399            AgentOutcome::Ok(_) => {}
4400            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4401            AgentOutcome::Quota(o) => {
4402                self.state.quota.push(QuotaLoss {
4403                    seat: seat.key.clone(),
4404                    node: "graft".to_owned(),
4405                    at: Timestamp::now(),
4406                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4407                });
4408                why = Some("rate limited (quota)".to_owned());
4409            }
4410            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4411                why = Some(format!(
4412                    "the fixer failed: {}",
4413                    e.lines()
4414                        .next()
4415                        .unwrap_or("")
4416                        .chars()
4417                        .take(160)
4418                        .collect::<String>()
4419                ));
4420            }
4421        }
4422        let agent_id = seat.agent.clone();
4423        self.state.seats.insert(seat.key.clone(), seat);
4424
4425        let mut post = None;
4426        let mut lines = None;
4427        let _coordination = self.coordinate(&wt).await?;
4428        if why.is_none() {
4429            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4430            if let Ok(r) = &rescued {
4431                self.state.note_withheld("graft", &r.withheld);
4432            }
4433            let head = git::rev_parse(&wt, "HEAD").await?;
4434            // What is measured must be what gets reviewed and gated: a failed
4435            // rescue, or anything still uncommitted (withheld lockfiles
4436            // aside, which are never part of the tree), rejects the graft.
4437            let dirty = match &rescued {
4438                Ok(r) => git::status_porcelain(&wt)
4439                    .await
4440                    .map(|st| {
4441                        st.lines().any(|l| {
4442                            !r.withheld
4443                                .iter()
4444                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4445                        })
4446                    })
4447                    .unwrap_or(true),
4448                Err(_) => true,
4449            };
4450            if dirty {
4451                why = Some("the fixer left changes that could not be committed".to_owned());
4452            } else if head == pre {
4453                why = Some("the fixer changed nothing".to_owned());
4454            } else {
4455                lines = git::changed_lines(&wt, &pre, &head).await;
4456                post = Some(head);
4457            }
4458        }
4459        let (adoption, adoption_error) = match &out {
4460            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4461            _ => (None, None),
4462        };
4463        let requested = graft::requested(&plan);
4464        let outcome = match why {
4465            None => {
4466                self.state.event(
4467                    "graft",
4468                    format!(
4469                        "applied: {}",
4470                        lines.map_or_else(
4471                            || "changed lines unavailable".to_owned(),
4472                            |n| format!("{n} changed lines")
4473                        )
4474                    ),
4475                );
4476                GraftOutcome::Applied
4477            }
4478            Some(why) => {
4479                // Not settled until the restore is confirmed: the record
4480                // stays `Running`, so a resume retries it.
4481                crate::recovery::restore_repair(&wt, &pre)
4482                    .await
4483                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4484                self.state.event(
4485                    "graft",
4486                    format!("rolled back: {why}; the winner is untouched"),
4487                );
4488                GraftOutcome::RolledBack(why)
4489            }
4490        };
4491        self.state.graft = Some(GraftRecord {
4492            pre,
4493            post,
4494            lines,
4495            limit: None,
4496            agent: Some(agent_id),
4497            outcome,
4498            requested,
4499            adoption,
4500            adoption_error,
4501        });
4502        self.state.save()?;
4503        Ok(())
4504    }
4505
4506    // ----------------------------------------------------------------- fold
4507
4508    async fn fold_losers(&mut self) -> Result<()> {
4509        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4510            return Ok(());
4511        };
4512        if !self.state.candidate_preservation.is_empty() {
4513            bail!("candidate preservation incomplete; refusing cleanup");
4514        }
4515        let repo = self.state.repo.clone();
4516        let _coordination = self.coordinate(&repo).await?;
4517        let mut folded = Vec::new();
4518        for i in 0..self.state.candidates.len() {
4519            let c = &self.state.candidates[i];
4520            if c.label == winner || c.folded {
4521                continue;
4522            }
4523            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4524            remove_preserved_worktree(&repo, &wt).await?;
4525            if git::branch_exists(&repo, &branch).await?
4526                && !git::branch_delete(&repo, &branch).await?
4527            {
4528                bail!("candidate branch removal refused");
4529            }
4530            self.state.candidates[i].folded = true;
4531            self.state.save()?;
4532            folded.push(label.to_string());
4533        }
4534        // The judges are finished; their checkouts are pure cost from here.
4535        let root = self.state.worktree_root();
4536        for j in 1..=self.roles.judges.len() {
4537            let wt = root.join(format!("judge-{j}"));
4538            if wt.exists() {
4539                remove_preserved_worktree(&repo, &wt).await?;
4540            }
4541        }
4542        // The design-deliberation stage is finished by the time a tally
4543        // exists — same reasoning as the judges above.
4544        if self.state.config.graph.advise {
4545            for k in 1..=self.state.config.graph.advisors {
4546                let wt = root.join(format!("advisor-{k}"));
4547                if wt.exists() {
4548                    remove_preserved_worktree(&repo, &wt).await?;
4549                }
4550            }
4551        }
4552        if !folded.is_empty() {
4553            self.state
4554                .event("fold", format!("folded candidates {}", folded.join(", ")));
4555            self.state.save()?;
4556        }
4557        Ok(())
4558    }
4559
4560    // ------------------------------------------------------------ base sync
4561
4562    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4563    /// anything verifies it.
4564    ///
4565    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4566    /// read whatever is checked out in the winner's worktree. Left alone that
4567    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4568    /// it when the run *branched* - and a run takes long enough that the base
4569    /// has usually moved by the time it gets here. A gate that ran there
4570    /// answers "green on the commit this run started from", not "green on
4571    /// what is about to land", and the difference showed up three times in
4572    /// one day as a green run whose merge would have reverted a file another
4573    /// pull request had already landed.
4574    ///
4575    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4576    /// `land::Step::Rebase` calls, rather than a second implementation of the
4577    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4578    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4579    /// doc warns about ("two answers to a question nobody notices until a
4580    /// diff is wrong").
4581    ///
4582    /// A conflict is not the end of the road: the standing rebase is handed
4583    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4584    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4585    /// land). Once it finishes, review and the gate run as usual on the
4586    /// rebased tree, which is where a breakage the new base caused is caught
4587    /// by the ordinary gate-fix round. magi resolves nothing itself.
4588    ///
4589    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4590    /// `state.base_sync.attempts`, is how many times the base is *rebased
4591    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4592    /// *conflict* was given to a fixer. When the fixer cannot finish the
4593    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4594    /// what was tried (rounds spent, paths still conflicted) and the branch
4595    /// and worktree stay exactly as they were - untouched, for a person to
4596    /// look at - which is also what makes re-entering this function
4597    /// afterwards a no-op instead of a second attempt at the same wall. A
4598    /// push failure ends the same way.
4599    async fn sync_to_base(&mut self) -> Result<()> {
4600        if self.state.status == RunStatus::AlreadyInBase {
4601            return Ok(());
4602        }
4603        let conflicted = self
4604            .state
4605            .base_sync
4606            .as_ref()
4607            .is_some_and(|s| s.conflict.is_some());
4608        let Some(winner) = self.state.winner().cloned() else {
4609            return Ok(());
4610        };
4611
4612        let repo = self.state.repo.clone();
4613        let remote = self.state.config.merge.remote.clone();
4614        let base_branch = self.state.base_branch.clone();
4615        let tracking = format!("{remote}/{base_branch}");
4616
4617        git::fetch(&repo, &remote, &base_branch).await.ok();
4618        // No network, or the remote never had this branch: the run already
4619        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4620        // otherwise), and one that got this far is not blocked by a fetch
4621        // that fails now. It just has no newer tip to compare against.
4622        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4623            return Ok(());
4624        };
4625
4626        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4627        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4628        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4629
4630        // Before any rebase, and before a recorded conflict is honoured: a
4631        // branch whose change reached the base under other commit ids has
4632        // nothing to rebase and nothing to conflict with, and a run that
4633        // already stopped on that phantom conflict recovers here on resume.
4634        // `behind == 0` with head == tip is a branch the base has since taken
4635        // in whole, whether or not a conflict was ever recorded: the ancestry
4636        // proof must still run (`classify` ignores a head still on the start
4637        // commit).
4638        if (behind > 0 || conflicted || head == tip)
4639            && self
4640                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4641                .await?
4642        {
4643            return Ok(());
4644        }
4645        if conflicted {
4646            return Ok(());
4647        }
4648
4649        if behind == 0 {
4650            // A fixer-finished rebase moves the branch ref before the
4651            // winner's worktree is told (`sync_to_head` below). A run that
4652            // died in between resumes here with `behind == 0` and a tree still
4653            // holding the pre-rebase files, which review and the gate would
4654            // then read. That state is exactly: HEAD moved off the tip the
4655            // rebase started from, yet the tree is still identical to that
4656            // tip. A tree with edits of its own differs from it, so nothing
4657            // is thrown away.
4658            if let Some(from) = self
4659                .state
4660                .rebase_fixes
4661                .iter()
4662                .rev()
4663                .find_map(|r| r.from.clone())
4664                && from != head
4665                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4666                    .await
4667                    .is_ok_and(|o| o.ok())
4668            {
4669                git::sync_to_head(&winner.worktree).await?;
4670            }
4671            // An earlier attempt may have rebased the branch locally and died
4672            // before pushing it (only the fresh-rebase arm below pushes).
4673            // Publish it now, so the plain push at PR time is not refused as
4674            // a non-fast-forward. A run already holding a recorded conflict
4675            // never reaches here; that case is out of scope.
4676            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4677            if let Some(why) = &conflict {
4678                self.state.status = RunStatus::Blocked;
4679                self.state.event("land", why.clone());
4680            }
4681            self.state.base_sync = Some(BaseSync {
4682                tip,
4683                behind: 0,
4684                attempts,
4685                conflict,
4686                already_in: None,
4687            });
4688            self.state.save()?;
4689            return Ok(());
4690        }
4691
4692        if attempts >= BASE_SYNC_ROUNDS {
4693            let why = format!(
4694                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4695                 rebase(s); rebasing again would only race it",
4696                winner.branch
4697            );
4698            self.state.status = RunStatus::Blocked;
4699            self.state.base_sync = Some(BaseSync {
4700                tip,
4701                behind,
4702                attempts,
4703                conflict: Some(why.clone()),
4704                already_in: None,
4705            });
4706            self.state.event("land", why);
4707            self.state.save()?;
4708            return Ok(());
4709        }
4710
4711        self.state.event(
4712            "land",
4713            format!(
4714                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4715                winner.branch
4716            ),
4717        );
4718        self.state.save()?;
4719
4720        // The remote's copy of the branch, read now and only if the fetch
4721        // really succeeded (a stale tracking ref must never pin a lease). It is
4722        // pushed over after a rebase only when it is a commit this branch
4723        // already contains, by ancestry or by patch (an earlier rebase of ours
4724        // that never reached the remote): anything else is somebody else's work.
4725        let branch_tracking = format!("{remote}/{}", winner.branch);
4726        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4727        let remote_tip = if self.state.task_pr_branch().is_none()
4728            && matches!(&fetched_branch, Ok(o) if o.ok())
4729        {
4730            git::rev_parse(&repo, &branch_tracking).await.ok()
4731        } else {
4732            None
4733        };
4734        // A remote tip this branch does not contain is somebody else's work:
4735        // rebasing would leave a local tip that can never be pushed. Stop
4736        // before touching anything and say so.
4737        if let Some(theirs) = &remote_tip
4738            && !git::is_ancestor(&repo, theirs, &head).await
4739            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4740                .await
4741                .is_ok_and(|missing| missing.is_empty())
4742        {
4743            let why = format!(
4744                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4745                 them",
4746                short(theirs),
4747                winner.branch
4748            );
4749            self.state.status = RunStatus::Blocked;
4750            self.state.base_sync = Some(BaseSync {
4751                tip,
4752                behind,
4753                attempts,
4754                conflict: Some(why.clone()),
4755                already_in: None,
4756            });
4757            self.state.event("land", why);
4758            self.state.save()?;
4759            return Ok(());
4760        }
4761
4762        let scratch = self.state.dir().join("base-sync");
4763        let rebased = match crate::rebase::rebase_with_fixer(
4764            &mut self.state,
4765            &scratch,
4766            &winner.branch,
4767            &tracking,
4768        )
4769        .await
4770        {
4771            Ok(crate::rebase::Rebased::Applied) => Ok(None),
4772            Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4773            Err(e) => Err(e),
4774        };
4775        let attempts = attempts + 1;
4776        match rebased {
4777            Ok(None) => {
4778                // The branch ref moved, but a worktree that already had it
4779                // checked out (the winner's) was not told; sync its index and
4780                // files before anything reads them.
4781                git::sync_to_head(&winner.worktree).await?;
4782                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4783                let mut conflict = None;
4784                if let Some(pinned) = &remote_tip {
4785                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4786                    match pushed {
4787                        Ok(o) if o.ok() => self.state.event(
4788                            "land",
4789                            format!("pushed rebased {} to {remote}", winner.branch),
4790                        ),
4791                        Ok(o) => {
4792                            conflict = Some(format!(
4793                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
4794                                winner.branch,
4795                                short(pinned),
4796                                o.stderr.chars().take(600).collect::<String>()
4797                            ));
4798                        }
4799                        Err(e) => {
4800                            conflict = Some(format!(
4801                                "rebased {} locally but could not push it: {e:#}",
4802                                winner.branch
4803                            ));
4804                        }
4805                    }
4806                }
4807                if let Some(why) = &conflict {
4808                    self.state.status = RunStatus::Blocked;
4809                    self.state.event("land", why.clone());
4810                }
4811                self.state.base_sync = Some(BaseSync {
4812                    tip: tip.clone(),
4813                    behind: 0,
4814                    attempts,
4815                    conflict,
4816                    already_in: None,
4817                });
4818                self.state
4819                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
4820            }
4821            Ok(Some(conflict)) => {
4822                let why = format!(
4823                    "{} conflicts with {tracking} and did not rebase: {}",
4824                    winner.branch,
4825                    conflict.chars().take(600).collect::<String>()
4826                );
4827                self.state.status = RunStatus::Blocked;
4828                self.state.base_sync = Some(BaseSync {
4829                    tip,
4830                    behind,
4831                    attempts,
4832                    conflict: Some(why.clone()),
4833                    already_in: None,
4834                });
4835                self.state.event("land", why);
4836            }
4837            // A refused Git write or a busy repository: nothing is wrong with
4838            // the branch. The scratch rebase stays standing and the run
4839            // resumes into it (`base_sync` must not claim a conflict).
4840            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4841                self.state.save()?;
4842                return Ok(());
4843            }
4844            Err(e) => {
4845                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4846                self.state.status = RunStatus::Blocked;
4847                self.state.base_sync = Some(BaseSync {
4848                    tip,
4849                    behind,
4850                    attempts,
4851                    conflict: Some(why.clone()),
4852                    already_in: None,
4853                });
4854                self.state.event("land", why);
4855            }
4856        }
4857        self.state.save()?;
4858        Ok(())
4859    }
4860
4861    /// Push a branch an earlier attempt rebased locally but never published,
4862    /// pinned to the remote tip read right after a successful fetch. Returns
4863    /// the reason when the run must stop; `None` when there was nothing to do
4864    /// (no remote copy, the same tip, or a remote copy this branch already
4865    /// contains, which the PR-time push fast-forwards) or the push succeeded.
4866    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4867        // Task competitions publish only after the PR text gate, to their
4868        // public head; never resurrect a candidate branch on the remote.
4869        if self.state.task_pr_branch().is_some() {
4870            return None;
4871        }
4872        let repo = self.state.repo.clone();
4873        let remote = self.state.config.merge.remote.clone();
4874        let fetched = git::fetch(&repo, &remote, branch).await;
4875        if !matches!(&fetched, Ok(o) if o.ok()) {
4876            return None;
4877        }
4878        let branch_tracking = format!("{remote}/{branch}");
4879        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4880        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4881            return None;
4882        }
4883        if !crate::reconcile::origin_missing(&repo, head, &theirs)
4884            .await
4885            .is_ok_and(|missing| missing.is_empty())
4886        {
4887            return Some(format!(
4888                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4889                 them",
4890                short(&theirs)
4891            ));
4892        }
4893        match git::push_pinned(&repo, &remote, branch, &theirs).await {
4894            Ok(o) if o.ok() => {
4895                self.state
4896                    .event("land", format!("pushed rebased {branch} to {remote}"));
4897                None
4898            }
4899            Ok(o) => Some(format!(
4900                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4901                 someone may have pushed): {}",
4902                short(&theirs),
4903                o.stderr.chars().take(600).collect::<String>()
4904            )),
4905            Err(e) => Some(format!(
4906                "{branch} is rebased locally but could not be pushed: {e:#}"
4907            )),
4908        }
4909    }
4910
4911    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
4912    /// is already on `tip` under other commit ids ([`crate::already`]); returns
4913    /// whether it did.
4914    ///
4915    /// Checked only when the base is ahead of the branch. A failing check is
4916    /// "not proven" - the ordinary rebase path then decides - never a reason to
4917    /// stop the run.
4918    ///
4919    /// The remote copy of the branch is held to the same standard as the local
4920    /// one: if it carries a tip this worktree does not, that tip must itself be
4921    /// proven in the base, or nothing is settled (a pull request would
4922    /// otherwise be closed over commits nobody checked). The pull request is
4923    /// closed *before* the terminal status is saved; if that fails for a
4924    /// reason other than a refusal (no network, a `gh` error) the run is left
4925    /// `Blocked` with the reason as its conflict, which a resume retries -
4926    /// the same recovery a phantom conflict gets.
4927    async fn settle_already_in(
4928        &mut self,
4929        branch: &str,
4930        tip: &str,
4931        head: &str,
4932        attempts: usize,
4933        behind: usize,
4934    ) -> Result<bool> {
4935        let repo = self.state.repo.clone();
4936        let remote = self.state.config.merge.remote.clone();
4937        let start = self.state.base_commit.clone();
4938        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4939            Ok(Some(e)) => e,
4940            Ok(None) => return Ok(false),
4941            Err(e) => {
4942                tracing::warn!("already-in-base check for {branch}: {e:#}");
4943                return Ok(false);
4944            }
4945        };
4946        let mut verified = vec![head.to_owned()];
4947        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4948        let fetched = git::fetch(&repo, &remote, &public_branch).await;
4949        if matches!(&fetched, Ok(o) if o.ok())
4950            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4951            && theirs != head
4952        {
4953            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4954                Ok(Some(_)) => verified.push(theirs),
4955                _ => return Ok(false),
4956            }
4957        }
4958        let base_branch = self.state.base_branch.clone();
4959        let message = format!(
4960            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4961             land",
4962            evidence.names(),
4963            evidence.proof.as_str()
4964        );
4965        let closed =
4966            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4967                .await;
4968        match closed {
4969            Ok(Ok(url)) => self
4970                .state
4971                .event("land", format!("closed {url}: superseded on {base_branch}")),
4972            Ok(Err(why)) => self
4973                .state
4974                .event("land", format!("did not close a pull request: {why}")),
4975            Err(e) => {
4976                let why = format!(
4977                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
4978                     not be closed ({e:#}); resume to retry"
4979                );
4980                self.state.status = RunStatus::Blocked;
4981                self.state.base_sync = Some(BaseSync {
4982                    tip: tip.to_owned(),
4983                    behind,
4984                    attempts,
4985                    conflict: Some(why.clone()),
4986                    already_in: None,
4987                });
4988                self.state.event("land", why);
4989                self.state.save()?;
4990                return Ok(true);
4991            }
4992        }
4993        self.state.status = RunStatus::AlreadyInBase;
4994        self.state.base_sync = Some(BaseSync {
4995            tip: tip.to_owned(),
4996            behind,
4997            attempts,
4998            conflict: None,
4999            already_in: Some(evidence),
5000        });
5001        self.state.event("land", message);
5002        self.state.save()?;
5003        self.settle_questions();
5004        Ok(true)
5005    }
5006
5007    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
5008    /// last landed the winner on, once it has run, else the commit the run
5009    /// branched from.
5010    ///
5011    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
5012    /// and `vote` all happen before there is a winner to rebase, so they
5013    /// compare every candidate against the branch point on purpose, and a
5014    /// base that moves after they are already done cannot change an answer
5015    /// they already gave.
5016    fn landing_base(&self) -> String {
5017        self.state
5018            .base_sync
5019            .as_ref()
5020            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5021    }
5022
5023    // ------------------------------------------------------- operator fix
5024
5025    /// Route specific, already-recorded review findings to a fixer for a
5026    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5027    /// entry point.
5028    ///
5029    /// Distinct from `review_loop`'s own fix step in three ways: it never
5030    /// runs a reviewer wave, it never spends review-round budget, and what
5031    /// happened is recorded as an [`OperatorFixRequest`] appended to
5032    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5033    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5034    /// and vote must never be rewritten to look like a manufactured blocking
5035    /// verdict.
5036    ///
5037    /// Only meaningful once review has actually concluded: `Ready` (handed
5038    /// off with findings still open, or simply concluded clean while minor
5039    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5040    /// gate failed). Everything else is refused: a run still in progress
5041    /// should simply be resumed, and a `Merged` run's branch has already
5042    /// landed — reopening *this* run's own record cannot change that, so the
5043    /// answer there is a fresh `magi review <branch>`.
5044    ///
5045    /// A real commit here re-verifies through a fresh, ordinary review-only
5046    /// run on the same branch ([`Self::review`]) rather than reopening this
5047    /// run's own `review_loop`: once any round in this run's history went
5048    /// clean, `review_conclusion` treats that as permanent by design (the
5049    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5050    /// way to force one more genuine reviewer wave out of *this* run without
5051    /// either rewriting history or weakening that guarantee for every other
5052    /// caller. A review-only run costs nothing extra — no implementation, no
5053    /// judging, no vote — and exercises the exact same review → verify →
5054    /// gate → (human) merge path, unmodified.
5055    pub async fn fix_selected(
5056        &mut self,
5057        ids: &[String],
5058        reason: &str,
5059        allow_stale: bool,
5060    ) -> Result<()> {
5061        let reason = reason.trim();
5062        if reason.is_empty() {
5063            bail!("a fix request needs a reason — that is the operator's own record of why");
5064        }
5065        if ids.is_empty() {
5066            bail!("no finding id given");
5067        }
5068        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5069            bail!(
5070                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5071                 has already concluded — can be given a targeted fix. A run still \
5072                 in progress should simply be resumed; a `merged` run's branch has \
5073                 already landed, so its answer is a fresh `magi review <branch>`, \
5074                 not reopening this run's own record",
5075                self.state.id,
5076                self.state.status.as_str()
5077            );
5078        }
5079        let Some(winner) = self.state.winner().cloned() else {
5080            bail!("run {} has no winning candidate to fix", self.state.id);
5081        };
5082        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5083            bail!(
5084                "branch `{}` no longer exists; this run cannot be extended",
5085                winner.branch
5086            );
5087        }
5088        let home = crate::run::home();
5089        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5090            bail!(
5091                "run {} is currently being worked on by another magi process",
5092                self.state.id
5093            );
5094        }
5095        // Held for the rest of this call, including the follow-up review
5096        // below: two `magi fix` invocations against the same run must not
5097        // both reach the worktree manipulation further down, which would
5098        // otherwise race to remove and recreate the same directory — see
5099        // [`FixClaim`]'s own doc.
5100        let _claim = FixClaim::acquire(&self.state.dir())?;
5101
5102        // Resolve every id before spending anything — an unknown id refuses
5103        // the whole request rather than silently dropping it — and dedup
5104        // while keeping the operator's own order.
5105        let mut seen = BTreeSet::new();
5106        let mut findings = Vec::new();
5107        let mut missing = Vec::new();
5108        for id in ids {
5109            if !seen.insert(id.clone()) {
5110                continue;
5111            }
5112            match self.state.finding(id) {
5113                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5114                    id: f.id.clone(),
5115                    severity: f.severity,
5116                    reviewer_vote: rec.vote,
5117                    round: round.round,
5118                    round_head: round.head.clone(),
5119                    reviewer: rec.reviewer,
5120                    agent: rec.agent.clone(),
5121                    file: f.file.clone(),
5122                    line: f.line,
5123                    title: f.title.clone(),
5124                    detail: f.detail.clone(),
5125                    outcome: OperatorFixOutcome::Pending,
5126                }),
5127                None => missing.push(id.clone()),
5128            }
5129        }
5130        if !missing.is_empty() {
5131            bail!(
5132                "unknown finding id(s): {}; nothing was changed",
5133                missing.join(", ")
5134            );
5135        }
5136
5137        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5138        let stale_details: Vec<(String, String)> = findings
5139            .iter()
5140            .filter(|f| f.round_head != head_at_request)
5141            .map(|f| (f.id.clone(), f.round_head.clone()))
5142            .collect();
5143        let stale = !stale_details.is_empty();
5144        if stale && !allow_stale {
5145            bail!(
5146                "the branch has moved since some finding(s) were raised — {} — now \
5147                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5148                stale_details
5149                    .iter()
5150                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5151                    .collect::<Vec<_>>()
5152                    .join(", "),
5153                short(&head_at_request)
5154            );
5155        }
5156
5157        let request = OperatorFixRequest {
5158            requested_at: Timestamp::now(),
5159            reason: reason.to_owned(),
5160            findings,
5161            head_at_request: head_at_request.clone(),
5162            allow_stale,
5163            stale,
5164            fix: None,
5165            result_head: None,
5166            follow_up_review_run: None,
5167        };
5168        self.state.event(
5169            "fix",
5170            format!(
5171                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5172                request.findings.len(),
5173                request
5174                    .findings
5175                    .iter()
5176                    .map(|f| f.id.as_str())
5177                    .collect::<Vec<_>>()
5178                    .join(", "),
5179            ),
5180        );
5181        // Recorded now, before any worktree work or the fixer call itself —
5182        // and re-saved at each checkpoint below: a crash at any point after
5183        // this (mid fixer call, mid follow-up review) must not lose the fact
5184        // that this was requested, for which findings, and why. Everything
5185        // past this point reads and writes through `request_index` rather
5186        // than a local variable, since `request` itself is moved here.
5187        self.state.operator_fixes.push(request);
5188        self.state.save()?;
5189        let request_index = self.state.operator_fixes.len() - 1;
5190
5191        // A fresh, dedicated worktree for this one call, never the winner's
5192        // own worktree in place: that one may already be gone (folded away),
5193        // and reusing it in place would leave the branch checked out there
5194        // when the follow-up review below tries to check it out again. Freed
5195        // immediately after, either way — but only once confirmed clean:
5196        // `worktree_remove` is a `git worktree remove --force`, which would
5197        // otherwise discard uncommitted work left there by the operator or
5198        // another process before this had a chance to even look at it.
5199        let repo = self.state.repo.clone();
5200        let coordination = self.coordinate(&repo).await?;
5201        if winner.worktree.exists() {
5202            // Lockfiles a rescue commit withheld stay untracked on purpose and
5203            // are already recorded; they are not the operator's work to protect.
5204            let dirty = git::git(
5205                &winner.worktree,
5206                &["status", "--porcelain", "--untracked-files=all"],
5207            )
5208            .await?;
5209            let only_withheld = dirty.lines().all(|l| {
5210                l.strip_prefix("?? ")
5211                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5212            });
5213            if !only_withheld {
5214                bail!(
5215                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5216                     discard them first",
5217                    winner.worktree.display()
5218                );
5219            }
5220            git::worktree_remove(&self.state.repo, &winner.worktree)
5221                .await
5222                .ok();
5223        }
5224        let fix_worktree = self.state.worktree_root().join("operator-fix");
5225        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5226        git::git(
5227            &self.state.repo,
5228            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5229        )
5230        .await
5231        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5232        if !git::is_clean(&fix_worktree).await? {
5233            git::worktree_remove(&self.state.repo, &fix_worktree)
5234                .await
5235                .ok();
5236            bail!(
5237                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5238                winner.branch
5239            );
5240        }
5241        let repo_for_setup = self.state.repo.clone();
5242        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5243            Ok(()) => {
5244                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5245            }
5246            Err(e) => Err(e),
5247        };
5248        if let Err(e) = setup {
5249            git::worktree_remove(&self.state.repo, &fix_worktree)
5250                .await
5251                .ok();
5252            return Err(e);
5253        }
5254
5255        let run_id = self.state.id.clone();
5256        let prompts = self.state.config.prompts.clone();
5257        let language = self.state.config.graph.language.clone();
5258        let sessions = self.state.config.graph.sessions;
5259        let artifacts = agent::artifacts_dir(&self.state.dir());
5260        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5261            .findings
5262            .iter()
5263            .map(|f| Finding {
5264                id: f.id.clone(),
5265                severity: f.severity,
5266                file: f.file.clone(),
5267                line: f.line,
5268                title: f.title.clone(),
5269                detail: f.detail.clone(),
5270            })
5271            .collect();
5272        let fix_prompt = prompt::operator_fix_in(
5273            &self.state.instruction,
5274            &finding_list,
5275            reason,
5276            &stale_details,
5277            &head_at_request,
5278            &language,
5279            &self.state.config.graph.github_language,
5280        );
5281        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5282        drop(coordination);
5283        let (job, seat, out) = self
5284            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5285                prompt: fix_prompt.clone(),
5286                spec,
5287                seat,
5288                cwd: fix_worktree.clone(),
5289                timeout,
5290                allow_write: true,
5291                sessions,
5292                artifacts: artifacts.clone(),
5293                stem: "operator-fix".to_owned(),
5294                handover: None,
5295            })
5296            .await;
5297        let agent_id = seat.agent.clone();
5298
5299        let mut fix = FixRecord {
5300            agent: agent_id,
5301            addressed: Vec::new(),
5302            rejected: Vec::new(),
5303            notes: String::new(),
5304            committed: false,
5305            failed: None,
5306            duration_ms: 0,
5307            continuation: None,
5308        };
5309        let mut final_seat = seat.clone();
5310        match out {
5311            AgentOutcome::Ok(o) => {
5312                fix.duration_ms = o.duration_ms;
5313                let parsed = verdict::extract_json::<FixReport>(&o.text);
5314                let incomplete_reason = match &parsed {
5315                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5316                        "the reply parsed, but it reported a command whose own CLI \
5317                         never confirmed an exit status"
5318                            .to_owned(),
5319                    ),
5320                    Ok(_) => None,
5321                    Err(e) => Some(e.to_string()),
5322                };
5323                match incomplete_reason {
5324                    None => {
5325                        let report = parsed.expect("checked Ok above");
5326                        fix.addressed = report.addressed;
5327                        fix.rejected = report.rejected;
5328                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5329                    }
5330                    Some(reason) => {
5331                        let (resumed_seat, resolved, failure, cont) = self
5332                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5333                            .await;
5334                        fix.duration_ms += cont.cumulative_wait_ms;
5335                        fix.continuation = Some(cont);
5336                        final_seat = resumed_seat;
5337                        match resolved {
5338                            Some(report) => {
5339                                fix.addressed = report.addressed;
5340                                fix.rejected = report.rejected;
5341                                fix.notes =
5342                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5343                            }
5344                            None => fix.failed = failure,
5345                        }
5346                    }
5347                }
5348            }
5349            AgentOutcome::Dropped(o) => {
5350                fix.duration_ms = o.duration_ms;
5351                let why = o
5352                    .dropped
5353                    .as_ref()
5354                    .map(|d| d.why.as_str())
5355                    .unwrap_or("the CLI ended the stream without delivering its answer");
5356                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5357            }
5358            AgentOutcome::Quota(o) => {
5359                self.state.quota.push(QuotaLoss {
5360                    seat: final_seat.key.clone(),
5361                    node: "fix".to_owned(),
5362                    at: Timestamp::now(),
5363                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5364                });
5365                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5366            }
5367            AgentOutcome::Failed(e) => fix.failed = Some(e),
5368            AgentOutcome::ResourceBlocked(e) => {
5369                fix.failed = Some(format!("the fixer was not started: {e}"));
5370            }
5371        }
5372        if fix.continuation.is_none() {
5373            fix.continuation = Some(ContinuationRecord::not_needed());
5374        }
5375        self.state.seats.insert(final_seat.key.clone(), final_seat);
5376
5377        let rescue_message = format!(
5378            "magi: operator-selected fix ({}) (uncommitted work)",
5379            self.state.operator_fixes[request_index]
5380                .findings
5381                .iter()
5382                .map(|f| f.id.as_str())
5383                .collect::<Vec<_>>()
5384                .join(", ")
5385        );
5386        let repo = self.state.repo.clone();
5387        let coordination = self.coordinate(&repo).await?;
5388        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5389            self.state.note_withheld("fix", &r.withheld);
5390        }
5391        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5392        fix.committed = after != head_at_request;
5393        git::worktree_remove(&self.state.repo, &fix_worktree)
5394            .await
5395            .ok();
5396        drop(coordination);
5397
5398        self.state.event(
5399            "fix",
5400            match &fix.failed {
5401                Some(reason) => format!(
5402                    "operator fix: adoption report was lost ({reason}); {}",
5403                    if fix.committed {
5404                        "committed"
5405                    } else {
5406                        "NO new commit"
5407                    }
5408                ),
5409                None => format!(
5410                    "operator fix: {} addressed, {} rejected, {}",
5411                    fix.addressed.len(),
5412                    fix.rejected.len(),
5413                    if fix.committed {
5414                        "committed"
5415                    } else {
5416                        "NO new commit"
5417                    }
5418                ),
5419            },
5420        );
5421
5422        // Every selected finding gets an outcome — never left `Pending` once
5423        // the fixer's own turn is over. A report that never came back at all
5424        // marks every one of them `Unreported`, not silently "not addressed":
5425        // quota, a dropped stream, or an exhausted continuation are gaps in
5426        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5427        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5428        for f in &mut self.state.operator_fixes[request_index].findings {
5429            f.outcome = if fix.failed.is_some() {
5430                OperatorFixOutcome::Unreported
5431            } else if fix.addressed.contains(&f.id) {
5432                OperatorFixOutcome::Addressed
5433            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5434                OperatorFixOutcome::Rejected { why: r.why.clone() }
5435            } else {
5436                OperatorFixOutcome::Unreported
5437            };
5438        }
5439
5440        let committed = fix.committed;
5441        if committed {
5442            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5443        }
5444        self.state.operator_fixes[request_index].fix = Some(fix);
5445        // Saved again now that the fixer's own outcome is final, on top of
5446        // the save right after the request was first pushed above.
5447        self.state.save()?;
5448
5449        if committed {
5450            self.state.event(
5451                "fix",
5452                format!(
5453                    "operator fix committed {}; opening a follow-up review-only run",
5454                    short(&after)
5455                ),
5456            );
5457            // The operator asked for the fix, and the follow-up serves whatever
5458            // task the run it follows served.
5459            let origin =
5460                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5461            match Self::review(
5462                &self.state.repo,
5463                &winner.branch,
5464                self.state.config.clone(),
5465                origin,
5466            )
5467            .await
5468            {
5469                Ok(mut follow_up) => {
5470                    follow_up.state.event(
5471                        "start",
5472                        format!(
5473                            "requested by an operator fix on run {} for finding(s) {}",
5474                            self.state.id,
5475                            self.state.operator_fixes[request_index]
5476                                .findings
5477                                .iter()
5478                                .map(|f| f.id.as_str())
5479                                .collect::<Vec<_>>()
5480                                .join(", "),
5481                        ),
5482                    );
5483                    follow_up.state.save()?;
5484                    let follow_up_id = follow_up.state.id.clone();
5485                    // The follow-up is a run like any other: it belongs to the
5486                    // task of the run it follows, or to one filed for it.
5487                    let adopted = match crate::direct::adopt(
5488                        &follow_up.state,
5489                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5490                    ) {
5491                        Ok(a) => a,
5492                        Err(e) => {
5493                            // An ownerless run must not spend agent calls; it
5494                            // stays saved, and `magi run --resume` adopts it.
5495                            self.state.event(
5496                                "fix",
5497                                format!(
5498                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5499                                ),
5500                            );
5501                            self.state.save()?;
5502                            return Ok(());
5503                        }
5504                    };
5505                    let executed = follow_up.execute().await;
5506                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5507                    if let Some(a) = adopted {
5508                        a.finish(&follow_up.state, executed);
5509                    }
5510                    if let Some(e) = failure {
5511                        self.state.event(
5512                            "fix",
5513                            format!(
5514                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5515                            ),
5516                        );
5517                    }
5518                    self.state.operator_fixes[request_index].follow_up_review_run =
5519                        Some(follow_up_id);
5520                }
5521                Err(e) => {
5522                    self.state.event(
5523                        "fix",
5524                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5525                    );
5526                }
5527            }
5528            self.state.save()?;
5529        }
5530
5531        Ok(())
5532    }
5533
5534    // --------------------------------------------------------------- review
5535
5536    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5537    /// names, in order, each at most once, moving on only when the call
5538    /// advances (an error, a quota hit or nothing usable - the same decision
5539    /// point `agent::chain_advances` is for the other chained roles).
5540    ///
5541    /// Each agent gets its own seat from `Runner::seat`: the same agent
5542    /// continues its conversation (the winner's own implementer seat when it
5543    /// is the winner's author, now that the competition is over), another
5544    /// takes a fresh one so the full prompt is sent again. A handover is
5545    /// recorded under `node`, which is what makes the fallback stick for the
5546    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5547    /// is returned, so an exhausted chain reads exactly like a single failed
5548    /// fixer: one quota loss, the same wording, the same refund. An earlier
5549    /// attempt's edits are left in the tree and judged, with the final
5550    /// attempt's, by what git says afterwards.
5551    ///
5552    /// Returns the job that produced the outcome, for `continue_fix_report`.
5553    async fn ask_fixer(
5554        &mut self,
5555        winner: &Candidate,
5556        node: &'static str,
5557        round: Option<usize>,
5558        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5559    ) -> (SeatJob, SeatState, AgentOutcome) {
5560        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5561        self.ask_fixer_among(attempts, node, round, None, build)
5562            .await
5563    }
5564
5565    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5566    /// worktree and a commit: before the next agent is asked after one that
5567    /// did not satisfy the call, the worktree is put back on that commit, so
5568    /// a failed agent's partial edits are not carried into the next one.
5569    async fn ask_fixer_among(
5570        &mut self,
5571        attempts: Vec<(AgentSpec, String)>,
5572        node: &'static str,
5573        round: Option<usize>,
5574        restore: Option<(&Path, &str)>,
5575        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5576    ) -> (SeatJob, SeatState, AgentOutcome) {
5577        let run_id = self.state.id.clone();
5578        let prompts = self.state.config.prompts.clone();
5579        let cache = self.state.config.cache_dir();
5580        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5581        let mut last = None;
5582        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5583            let seat = self.seat(&key, &spec.id);
5584            let mut job = build(spec.clone(), seat);
5585            if i > 0 {
5586                job.stem = format!("{}-{}", job.stem, spec.id);
5587            }
5588            let ctx = WaveCtx {
5589                carry_seats: false,
5590                pauses: Some((&self.pause, &self.interrupt)),
5591                run: &run_id,
5592                node,
5593                prompts: &prompts,
5594                cache: cache.as_deref(),
5595                round,
5596            };
5597            let (seat, out) =
5598                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5599            // A resource block never advances the chain: the chosen agent
5600            // was not asked, so it must be the one asked again on resume.
5601            let advances = match &out {
5602                AgentOutcome::Ok(o) => agent::output_advances(o),
5603                AgentOutcome::ResourceBlocked(_) => false,
5604                _ => true,
5605            };
5606            if let Some(next) = ids.get(i + 1)
5607                && advances
5608            {
5609                self.state.seats.insert(seat.key.clone(), seat.clone());
5610                let class =
5611                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5612                let reason = match &out {
5613                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5614                    other => fail_reason(other),
5615                };
5616                // A failed agent's edits must not reach the next one, and a
5617                // restore that did not work ends the chain: the call fails
5618                // and the caller's own (confirmed) restore decides.
5619                if let Some((wt, rev)) = restore
5620                    && let Err(e) = self.graft_restore(wt, rev).await
5621                {
5622                    let why =
5623                        format!("could not restore the worktree before the next fixer: {e:#}");
5624                    tracing::warn!("{why}");
5625                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5626                        AgentOutcome::ResourceBlocked(why)
5627                    } else {
5628                        AgentOutcome::Failed(why)
5629                    };
5630                    last = Some((job, seat, out));
5631                    break;
5632                }
5633                record_handover(
5634                    &mut self.state,
5635                    node,
5636                    &seat.key,
5637                    &spec.id,
5638                    next,
5639                    &class,
5640                    &reason,
5641                );
5642                continue;
5643            }
5644            last = Some((job, seat, out));
5645            break;
5646        }
5647        last.expect("the fixer chain always has an entry")
5648    }
5649
5650    /// The one repair a verification-only failure earns (see
5651    /// [`crate::recovery`]), before the first review round of a run that took
5652    /// a branch over.
5653    ///
5654    /// Saved as started *before* the fixer is called, so a resume never
5655    /// repairs twice; a start found on re-entry is rolled back to the commit
5656    /// it began from. A failed or unusable fixer leaves the tree as it was and
5657    /// the review goes on over it, so the first round's own verification
5658    /// reports the failure again. The failure is never carried into a round as
5659    /// a result: verification is re-run on whatever commit this leaves.
5660    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5661        use crate::recovery::{Mode, Repair};
5662        if !self.state.reviews.is_empty() {
5663            return Ok(());
5664        }
5665        let Some(mut rec) = self.state.recovery.clone() else {
5666            return Ok(());
5667        };
5668        if rec.mode != Mode::VerifyRepair {
5669            return Ok(());
5670        }
5671        let wt = winner.worktree.clone();
5672        let mut head = git::rev_parse(&wt, "HEAD").await?;
5673        match rec.repair.clone() {
5674            Repair::Done { .. } => return Ok(()),
5675            Repair::WaitingForRestore { before } => {
5676                self.graft_restore(&wt, &before).await?;
5677                head = before;
5678                rec.repair = Repair::NotStarted;
5679                self.state.recovery = Some(rec.clone());
5680                self.state.save()?;
5681            }
5682            Repair::Started { before } => {
5683                // Interrupted: the unfinished edits are not trusted.
5684                // An unconfirmed rollback blocks the run with `Started` kept,
5685                // so a resume retries it; the tree is never reviewed as is.
5686                if let Err(e) = self.graft_restore(&wt, &before).await {
5687                    if e.is::<crate::repo_lock::ResourceBlocked>() {
5688                        return Err(e);
5689                    }
5690                    return self
5691                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5692                        .await;
5693                }
5694                rec.repair = Repair::Done {
5695                    before: before.clone(),
5696                    after: before,
5697                    committed: false,
5698                    failed: Some("interrupted; rolled back".to_owned()),
5699                };
5700                self.state
5701                    .event("recovery", "the repair was interrupted and rolled back");
5702                self.state.recovery = Some(rec);
5703                self.state.save()?;
5704                return Ok(());
5705            }
5706            Repair::NotStarted => {}
5707        }
5708        if crate::recovery::recheck(&mut rec, &head) {
5709            self.state.event("recovery", rec.summary());
5710            self.state.recovery = Some(rec);
5711            self.state.save()?;
5712            return Ok(());
5713        }
5714        rec.repair = Repair::Started {
5715            before: head.clone(),
5716        };
5717        self.state.recovery = Some(rec.clone());
5718        self.state.save()?;
5719
5720        let language = self.state.config.graph.language.clone();
5721        let sessions = self.state.config.graph.sessions;
5722        let artifacts = agent::artifacts_dir(&self.state.dir());
5723        let fix_prompt = prompt::recovery_fix(
5724            &self.state.instruction,
5725            &rec,
5726            self.state.config.graph.review_rounds,
5727            &language,
5728        );
5729        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5730        self.state.event(
5731            "recovery",
5732            format!(
5733                "repair: the fixer starts from the failed verification on {}",
5734                short(&head)
5735            ),
5736        );
5737        let (_job, seat, out) = self
5738            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5739                prompt: fix_prompt.clone(),
5740                spec,
5741                seat,
5742                cwd: wt.clone(),
5743                timeout,
5744                allow_write: true,
5745                sessions,
5746                artifacts: artifacts.clone(),
5747                stem: "fix-recovery".to_owned(),
5748                handover: None,
5749            })
5750            .await;
5751        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5752            rec.repair = Repair::WaitingForRestore { before: head };
5753            self.state.recovery = Some(rec);
5754            return self.stop_for_resource();
5755        }
5756        let failed = match &out {
5757            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5758            AgentOutcome::Quota(o) => {
5759                self.state.quota.push(QuotaLoss {
5760                    seat: seat.key.clone(),
5761                    node: "fix".to_owned(),
5762                    at: Timestamp::now(),
5763                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5764                });
5765                Some("rate limited (quota); the fixer could not run".to_owned())
5766            }
5767            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5768            AgentOutcome::Failed(e) => Some(e.clone()),
5769        };
5770        self.state.seats.insert(seat.key.clone(), seat);
5771        let coordination = self.coordinate(&wt).await?;
5772        let failed = match failed {
5773            // The agent's partial edits are not kept.
5774            Some(why) => {
5775                if let Err(e) = git::restore_to(&wt, &head).await {
5776                    return self
5777                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5778                        .await;
5779                }
5780                Some(why)
5781            }
5782            None => {
5783                if let Ok(r) = git::rescue_commit(
5784                    &wt,
5785                    "magi: recovery repair of the failed verification (uncommitted work)",
5786                )
5787                .await
5788                {
5789                    self.state.note_withheld("fix", &r.withheld);
5790                }
5791                None
5792            }
5793        };
5794        let after = git::rev_parse(&wt, "HEAD").await?;
5795        drop(coordination);
5796        self.state.event(
5797            "recovery",
5798            match &failed {
5799                Some(why) => format!("repair: nothing kept ({why})"),
5800                None if after != head => {
5801                    format!("repair: committed {}", short(&after))
5802                }
5803                None => "repair: the fixer made no commit".to_owned(),
5804            },
5805        );
5806        rec.repair = Repair::Done {
5807            before: head.clone(),
5808            committed: after != head,
5809            after,
5810            failed,
5811        };
5812        self.state.recovery = Some(rec);
5813        self.state.save()?;
5814        Ok(())
5815    }
5816
5817    /// A repair whose rollback could not be confirmed: the run is blocked and
5818    /// resumable, and the unconfirmed state is never reviewed.
5819    async fn stop_recovery(
5820        &mut self,
5821        mut rec: crate::recovery::RecoveryRecord,
5822        head: &str,
5823        why: String,
5824    ) -> Result<()> {
5825        // `Started` stays recorded: a resume retries the restore.
5826        rec.repair = crate::recovery::Repair::Started {
5827            before: head.to_owned(),
5828        };
5829        self.state.recovery = Some(rec);
5830        self.state
5831            .event("recovery", format!("repair could not be undone: {why}"));
5832        self.state.status = RunStatus::Blocked;
5833        self.state.save()?;
5834        bail!("recovery repair could not be rolled back: {why}")
5835    }
5836
5837    async fn review_loop(&mut self) -> Result<()> {
5838        // A base that would not rebase is a person's decision, not a review
5839        // round: nothing here would change the answer, and reviewers and a
5840        // fixer would be spending real budget on a tree that cannot land
5841        // regardless of what they find.
5842        if self
5843            .state
5844            .base_sync
5845            .as_ref()
5846            .is_some_and(|s| s.conflict.is_some())
5847        {
5848            return Ok(());
5849        }
5850        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
5851        // agent files with `magi task add` name the run that paid for it. The
5852        // prompt overlay is cloned alongside it because the waves borrow it
5853        // while `self` is mutably borrowed by the node's own bookkeeping.
5854        let run_id = self.state.id.clone();
5855        let prompts = self.state.config.prompts.clone();
5856        let Some(winner) = self.state.winner().cloned() else {
5857            return Ok(());
5858        };
5859        let max_rounds = self.state.config.graph.review_rounds;
5860        // A clean round, an exhausted round budget, or a stalled tree (see
5861        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
5862        // they are recorded — recomputed here, not read off `status`, so a
5863        // reentry into a run that already stopped restates the identical
5864        // verdict instead of silently handing back whatever an earlier node
5865        // in this same walk clobbered `status` to (a solo-candidate
5866        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
5867        // below runs an empty range once the budget is spent, and would
5868        // otherwise fall through without touching `status` at all.
5869        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5870            // A reentry after a crash between the last round's save and
5871            // `stop_reviewing` reaches the hand-off here, not there.
5872            if status == RunStatus::Gating {
5873                self.record_contested_handoff();
5874            }
5875            self.state.status = status;
5876            self.state.save()?;
5877            return Ok(());
5878        }
5879        self.state.status = RunStatus::Reviewing;
5880        // A last recorded round whose own verification never resolved
5881        // (`ResourceBlocked` — the shared build cache, not the patch) is
5882        // never a concluded round, whatever the round budget says: starting
5883        // a fresh round on top of it would spend a whole new reviewer wave
5884        // re-reading an unchanged patch instead of just retrying the one
5885        // check that actually needs it, and once the budget is spent the
5886        // loop below has nothing left to do at all (its range is empty).
5887        // Retry that check directly instead, exactly the same retry
5888        // `stop_reviewing` already does for its own catch-up case.
5889        if self
5890            .state
5891            .reviews
5892            .last()
5893            .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5894        {
5895            let shell = self.state.config.shell();
5896            return self
5897                .stop_reviewing(
5898                    "the last round's own verification never resolved",
5899                    &shell,
5900                    &winner.worktree,
5901                )
5902                .await;
5903        }
5904
5905        self.recovery_repair(&winner).await?;
5906        if self.resource_halt()? {
5907            return Ok(());
5908        }
5909
5910        let repo = self.state.repo.clone();
5911        let root = self.state.worktree_root();
5912        let language = self.state.config.graph.language.clone();
5913        let sessions = self.state.config.graph.sessions;
5914        let artifacts = self.merge_repair_artifacts();
5915        let base = self.landing_base();
5916        let base_short = short(&base);
5917        let reviewers = self.roles.reviewers.clone();
5918        let shell = self.state.config.shell();
5919
5920        for round in (self.state.reviews.len() + 1)..=max_rounds {
5921            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5922            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5923            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5924            // The prior round's own record, already persisted — never a
5925            // hand-carried variable of just its failing output: that is
5926            // exactly what let a round's e2e result drift out of sync with
5927            // which commit it was actually about (see `SCHEMA`'s doc for
5928            // schema 8). Judged against `head`, the commit reviewers are
5929            // about to look at now, so the summary always reads as "an
5930            // earlier head" here — this round's own patch has not been
5931            // checked yet.
5932            let prev_verification = self
5933                .state
5934                .reviews
5935                .last()
5936                .and_then(|r| r.verification_summary(&head));
5937            // Only the first round of a run that took a branch over: later
5938            // rounds have this run's own history.
5939            let prior_section = match (&self.state.recovery, round) {
5940                (Some(rec), 1) if self.state.reviews.is_empty() => {
5941                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5942                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5943                            Some(crate::recovery::head_of(
5944                                &git::diff(&winner.worktree, reviewed, "HEAD")
5945                                    .await
5946                                    .unwrap_or_default(),
5947                                crate::recovery::SINCE_DIFF_MAX,
5948                            ))
5949                        }
5950                        _ => None,
5951                    };
5952                    Some(prompt::recovery_review(rec, since.as_deref()))
5953                }
5954                _ => None,
5955            };
5956
5957            // Each reviewer gets its own detached checkout of exactly this
5958            // commit: nobody can perturb the winner's tree, and the fixer can
5959            // keep working without racing a reviewer.
5960            let mut jobs = Vec::new();
5961            let coordination = self.coordinate(&repo).await?;
5962            for (r, spec) in reviewers.iter().cloned().enumerate() {
5963                let wt = root.join(format!("review-{}", r + 1));
5964                // `clean -fdx` in a reset wipes setup's products, so every
5965                // round sets the seat up again.
5966                if wt.exists() {
5967                    git::reset_detached(&wt, &head).await?;
5968                } else {
5969                    git::worktree_add_detached(&repo, &wt, &head).await?;
5970                }
5971                self.setup_seat_worktree(&repo, &wt, false).await?;
5972                let seat_key = format!("review-{}", r + 1);
5973                // The seat starts the round on whoever answered it last, not
5974                // on the agent the spec names, so a failure is not re-paid.
5975                let spec = pick_start_spec(
5976                    &self.roles.reviewer_roster,
5977                    spec,
5978                    self.state.seat_history.get(&seat_key),
5979                );
5980                let seat = self.seat(&seat_key, &spec.id);
5981                jobs.push(SeatJob {
5982                    prompt: prompt::review(&prompt::ReviewCtx {
5983                        instruction: &self.state.instruction,
5984                        branch: &winner.branch,
5985                        base_short: &base_short,
5986                        stat: &stat,
5987                        patch: &patch,
5988                        verification: prev_verification.as_ref(),
5989                        prior: prior_section.as_deref(),
5990                        reviewers: reviewers.len(),
5991                        round,
5992                        rounds: max_rounds,
5993                        // A review-only run has no rankings, so nothing
5994                        // competed for this patch and the reviewer is told so.
5995                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5996                        lens: Lens::for_seat(r),
5997                        language: &language,
5998                        github_language: &self.state.config.graph.github_language,
5999                    }),
6000                    spec,
6001                    seat,
6002                    cwd: wt,
6003                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6004                    allow_write: false,
6005                    sessions,
6006                    artifacts: artifacts.clone(),
6007                    stem: format!("review-{round}-{}", r + 1),
6008                    handover: None,
6009                });
6010            }
6011
6012            drop(coordination);
6013            self.state.event(
6014                "review",
6015                format!(
6016                    "round {round}: {} reviewers on {}",
6017                    jobs.len(),
6018                    short(&head)
6019                ),
6020            );
6021            let mut quota_losses = Vec::new();
6022            let review_retries = self.state.config.graph.retries;
6023            let review_cache = self.state.config.cache_dir();
6024            let ctx = WaveCtx {
6025                carry_seats: true,
6026                pauses: Some((&self.pause, &self.interrupt)),
6027                run: &run_id,
6028                node: "review",
6029                prompts: &prompts,
6030                cache: review_cache.as_deref(),
6031                round: Some(round),
6032            };
6033            let results = ask_json_wave::<Review>(
6034                jobs,
6035                Arc::clone(&self.sem),
6036                review_retries,
6037                &self.roles.reviewer_roster,
6038                &ctx,
6039                &mut quota_losses,
6040                &mut self.state,
6041                &|_: &Review| Ok(()),
6042            )
6043            .await;
6044            if self.state.resource_block.is_some() {
6045                return self.stop_for_resource();
6046            }
6047            // Counted before the move below: how many of *this* round's
6048            // reviewer seats were lost to their own rate limit, as opposed to
6049            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6050            let round_quota_missing = quota_losses.len();
6051            self.state.quota.extend(quota_losses);
6052
6053            let mut records = Vec::new();
6054            let mut all_findings = Vec::new();
6055            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6056                let agent_id = seat.agent.clone();
6057                self.state.seats.insert(seat.key.clone(), seat);
6058                let mut record = ReviewRecord {
6059                    reviewer: r + 1,
6060                    agent: agent_id,
6061                    summary: String::new(),
6062                    findings: Vec::new(),
6063                    vote: None,
6064                    failed: None,
6065                    duration_ms: 0,
6066                    // Set for both outcomes: `failed: Some(_)` with
6067                    // `attempts > 0` is a seat every retry still lost, not a
6068                    // recovered one — only `failed: None` with `attempts > 0`
6069                    // reads as "answered after a nudge" (see this field's own
6070                    // doc).
6071                    attempts,
6072                };
6073                match res {
6074                    Ok((review, out)) => {
6075                        // Sanitized here, at the point every other piece of
6076                        // agent prose in this file is (candidate summaries,
6077                        // deliberation turns, vote reasons): a reviewer's own
6078                        // words are the one thing about it that could name
6079                        // it, and reconsideration below broadcasts this same
6080                        // summary and these same findings to every other
6081                        // seat on the panel.
6082                        record.summary =
6083                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6084                        record.vote = Some(review.vote);
6085                        record.duration_ms = out.duration_ms;
6086                        for (n, mut f) in review.findings.into_iter().enumerate() {
6087                            // ids are magi's, never the agent's: the fixer's
6088                            // adoption report is keyed by them.
6089                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6090                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6091                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6092                            // `file` is agent-supplied prose too, never
6093                            // checked against the real tree — the same
6094                            // exposure `title`/`detail` above have, just in
6095                            // a field easy to forget because it looks like a
6096                            // path rather than free text.
6097                            f.file = f
6098                                .file
6099                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6100                            all_findings.push(f.clone());
6101                            record.findings.push(f);
6102                        }
6103                        self.state.event(
6104                            "review",
6105                            format!(
6106                                "round {round}: reviewer {} voted {} with {} finding(s)",
6107                                r + 1,
6108                                review.vote.label(),
6109                                record.findings.len()
6110                            ),
6111                        );
6112                    }
6113                    Err(e) => {
6114                        record.failed = Some(e.to_string());
6115                        self.state.event(
6116                            "review",
6117                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6118                        );
6119                    }
6120                }
6121                records.push(record);
6122            }
6123
6124            // Tally the round's votes and, if they split, spend the one
6125            // round of reconsideration the split -> deliberate -> revote
6126            // shape `judge`/`vote` use for the panel, sized down to what a
6127            // read-only review round can afford: one round, and a revote
6128            // rather than an argument, because the panel already wrote its
6129            // reasoning down as findings the first time around.
6130            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6131            let vote_split =
6132                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6133            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6134            if vote_split {
6135                self.state.event(
6136                    "review",
6137                    format!(
6138                        "round {round}: votes split ({}) — one round of reconsideration",
6139                        initial_votes
6140                            .iter()
6141                            .map(|v| v.label())
6142                            .collect::<Vec<_>>()
6143                            .join(", ")
6144                    ),
6145                );
6146                // Seats read every seat's findings and votes, still numbered
6147                // and never named — the same anonymity `review` itself keeps.
6148                let panel: Vec<ReviewSeatReport<'_>> = records
6149                    .iter()
6150                    .filter_map(|r| {
6151                        r.vote.map(|vote| ReviewSeatReport {
6152                            reviewer: r.reviewer,
6153                            vote,
6154                            summary: &r.summary,
6155                            findings: &r.findings,
6156                        })
6157                    })
6158                    .collect();
6159
6160                let mut jobs = Vec::new();
6161                let mut seats_at = Vec::new();
6162                for (r, spec) in reviewers.iter().cloned().enumerate() {
6163                    // A seat with no initial vote has nothing to reconsider
6164                    // from and stays absent, the same as it stayed absent
6165                    // from `panel` above.
6166                    if records[r].vote.is_none() {
6167                        continue;
6168                    }
6169                    let wt = root.join(format!("review-{}", r + 1));
6170                    let seat_key = format!("review-{}", r + 1);
6171                    let spec = self.occupant(&seat_key, spec);
6172                    let seat = self.seat(&seat_key, &spec.id);
6173                    // A seat with no live session has already forgotten the
6174                    // initial review's prompt — restate the patch it is
6175                    // voting on, the same as `deliberate`/`vote` do for a
6176                    // judge in the same position.
6177                    // The panel already carries this seat's own review and
6178                    // vote, so restating the patch makes the prompt whole for
6179                    // a seat handed to another agent.
6180                    let build = |with_patch: bool| {
6181                        prompt::review_reconsider(&ReviewReconsiderCtx {
6182                            instruction: &self.state.instruction,
6183                            reviewer: r + 1,
6184                            lens: Lens::for_seat(r),
6185                            panel: &panel,
6186                            patch: with_patch.then_some(ReviewPatch {
6187                                branch: &winner.branch,
6188                                base_short: &base_short,
6189                                stat: &stat,
6190                                patch: &patch,
6191                            }),
6192                            round,
6193                            rounds: max_rounds,
6194                            language: &language,
6195                        })
6196                    };
6197                    let full = build(true);
6198                    let prompt = if has_context(&spec, &seat, sessions) {
6199                        build(false)
6200                    } else {
6201                        full.clone()
6202                    };
6203                    jobs.push(SeatJob {
6204                        prompt,
6205                        spec,
6206                        seat,
6207                        cwd: wt,
6208                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6209                        allow_write: false,
6210                        sessions,
6211                        artifacts: artifacts.clone(),
6212                        stem: format!("review-{round}-reconsider-{}", r + 1),
6213                        handover: Some(full),
6214                    });
6215                    seats_at.push(r);
6216                }
6217
6218                let mut recon_quota_losses = Vec::new();
6219                let recon_cache = self.state.config.cache_dir();
6220                let recon_ctx = WaveCtx {
6221                    carry_seats: true,
6222                    pauses: Some((&self.pause, &self.interrupt)),
6223                    run: &run_id,
6224                    node: "review",
6225                    prompts: &prompts,
6226                    cache: recon_cache.as_deref(),
6227                    round: Some(round),
6228                };
6229                let recon_results = ask_json_wave::<ReviewRevote>(
6230                    jobs,
6231                    Arc::clone(&self.sem),
6232                    review_retries,
6233                    &self.roles.reviewer_roster,
6234                    &recon_ctx,
6235                    &mut recon_quota_losses,
6236                    &mut self.state,
6237                    &|_: &ReviewRevote| Ok(()),
6238                )
6239                .await;
6240                if self.state.resource_block.is_some() {
6241                    return self.stop_for_resource();
6242                }
6243                self.state.quota.extend(recon_quota_losses);
6244
6245                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6246                    let agent_id = seat.agent.clone();
6247                    self.state.seats.insert(seat.key.clone(), seat);
6248                    let mut rec = ReviewRevoteRecord {
6249                        reviewer: r + 1,
6250                        agent: agent_id,
6251                        vote: None,
6252                        reason: String::new(),
6253                        failed: None,
6254                    };
6255                    match res {
6256                        Ok((rv, _)) => {
6257                            rec.vote = Some(rv.vote);
6258                            rec.reason =
6259                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6260                            self.state.event(
6261                                "review",
6262                                format!(
6263                                    "round {round}: reviewer {} revoted {}",
6264                                    r + 1,
6265                                    rv.vote.label()
6266                                ),
6267                            );
6268                        }
6269                        Err(e) => {
6270                            rec.failed = Some(e.to_string());
6271                            self.state.event(
6272                                "review",
6273                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6274                            );
6275                        }
6276                    }
6277                    reconsideration.push(rec);
6278                }
6279            } else if initial_votes.len() > 1 {
6280                self.state.event(
6281                    "review",
6282                    format!(
6283                        "round {round}: votes agreed ({}) — no reconsideration",
6284                        initial_votes[0].label()
6285                    ),
6286                );
6287            }
6288
6289            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6290            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6291            // A round that already has a blocking finding and a round left to
6292            // try is going back to the fixer no matter what `verify.e2e`
6293            // says, so running it first only spends the loop's slowest step
6294            // (minutes, for a Rust repo's full test suite) on a head about
6295            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6296            // runs once a round has no blocking findings left (see
6297            // `round_is_clean`, which a deferred — empty — `e2e` can never
6298            // satisfy since `blocking` is nonzero whenever this branch is
6299            // taken), and `stop_reviewing` forces a real run before it will
6300            // ever read a deferred round as green.
6301            let defer_e2e =
6302                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6303            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6304                let reason =
6305                    format!("{blocking} blocking finding(s) already required a fix this round");
6306                self.state.event(
6307                    "verify",
6308                    format!(
6309                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6310                         {}); it will run once a round has none left",
6311                        short(&head)
6312                    ),
6313                );
6314                (Vec::new(), false, true, Some(reason))
6315            } else {
6316                let e2e_commands = self.state.config.verify.e2e.clone();
6317                let cache_dir = self.state.config.cache_dir();
6318                let context = format!("round {round}");
6319                let wait = self.repository_wait();
6320                let (e2e, verify_retried) = with_cache_lease(
6321                    &mut self.state,
6322                    cache_dir.as_deref(),
6323                    "e2e",
6324                    "e2e",
6325                    &winner.worktree,
6326                    &head,
6327                    verify_timeout,
6328                    &context,
6329                    &wait,
6330                    |state, budget| {
6331                        let shell = shell.clone();
6332                        let e2e_commands = e2e_commands.clone();
6333                        let worktree = winner.worktree.clone();
6334                        let context = context.clone();
6335                        async move {
6336                            run_e2e_with_retry(
6337                                state,
6338                                &shell,
6339                                &e2e_commands,
6340                                &worktree,
6341                                budget,
6342                                &context,
6343                            )
6344                            .await
6345                        }
6346                    },
6347                )
6348                .await?;
6349                (e2e, verify_retried, false, None)
6350            };
6351
6352            let expected = records.len();
6353            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6354            let incomplete = answered < expected;
6355            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6356            let policy = self.state.config.graph.incomplete_review;
6357            let clean = round_is_clean(
6358                blocking,
6359                e2e_ok,
6360                answered,
6361                expected,
6362                round_quota_missing,
6363                policy,
6364            );
6365
6366            let mut round_record = ReviewRound {
6367                round,
6368                head: head.clone(),
6369                verified_head: None,
6370                verified_at: None,
6371                reviews: records,
6372                e2e,
6373                verify_retried,
6374                e2e_deferred,
6375                e2e_defer_reason,
6376                fix: None,
6377                blocking,
6378                answered,
6379                expected,
6380                clean,
6381                progressed: false,
6382                vote_split,
6383                reconsideration,
6384                verdict: None,
6385            };
6386            // The final vote per seat is its revote where reconsideration
6387            // ran and answered, its initial vote otherwise — the same
6388            // fallback `tally` uses for a judge whose private vote failed.
6389            round_record.verdict = ReviewVote::worst(
6390                round_record
6391                    .final_votes()
6392                    .into_iter()
6393                    .map(|(_, _, vote)| vote),
6394            );
6395            // Which commit and when magi actually attempted to check —
6396            // known the moment a command was dispatched against `head`,
6397            // whether or not it finished: a resource-blocked attempt still
6398            // targeted a specific commit at a specific time, and leaving
6399            // that unrecorded is exactly what made `verification_summary`
6400            // report a fresh attempt as "commit unknown ... recorded before
6401            // this was tracked", indistinguishable from a genuinely old,
6402            // untracked record. Only a deferred or unconfigured round never
6403            // ran at all and has nothing to record — see
6404            // `ReviewRound::verified_head`'s own doc.
6405            if !matches!(
6406                round_record.e2e_status(),
6407                E2eStatus::Deferred | E2eStatus::NotConfigured
6408            ) {
6409                round_record.verified_head = Some(head.clone());
6410                round_record.verified_at = Some(Timestamp::now());
6411            }
6412            let this_round_verification = round_record.verification_summary(&head);
6413
6414            if incomplete {
6415                let missing: Vec<String> = round_record
6416                    .reviews
6417                    .iter()
6418                    .filter(|r| r.failed.is_some())
6419                    .map(|r| format!("review-{}", r.reviewer))
6420                    .collect();
6421                self.state.event(
6422                    "review",
6423                    format!(
6424                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6425                        missing.join(", ")
6426                    ),
6427                );
6428            }
6429
6430            if clean {
6431                self.state.event(
6432                    "review",
6433                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6434                        format!(
6435                            "round {round}: clean (warn policy, incomplete panel) — no \
6436                             blocking findings from the seats that answered, verification green"
6437                        )
6438                    } else if incomplete {
6439                        format!(
6440                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6441                             quorum) — no blocking findings from the seats that answered, \
6442                             verification green",
6443                            expected - answered
6444                        )
6445                    } else {
6446                        format!("round {round}: clean — no blocking findings, verification green")
6447                    },
6448                );
6449                self.state.reviews.push(round_record);
6450                self.state.status = RunStatus::Gating;
6451                self.state.save()?;
6452                return Ok(());
6453            }
6454
6455            // Nothing was raised and verification passed, but not every seat
6456            // answered and `round_is_clean` still refused to call it clean —
6457            // either a seat is missing for a reason other than its own quota
6458            // (a crash, a timeout, unparsable output — worth another try), or
6459            // every seat that could have answered lost its quota and nobody
6460            // is left to decide on: re-review rather than send the fixer
6461            // after a round with nothing to fix.
6462            if incomplete && blocking == 0 && e2e_ok {
6463                self.state.reviews.push(round_record);
6464                self.state.save()?;
6465                if round == max_rounds {
6466                    self.state.status = RunStatus::Blocked;
6467                    self.state.event(
6468                        "review",
6469                        format!(
6470                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6471                             refusing to call it clean",
6472                            expected - answered
6473                        ),
6474                    );
6475                    return Ok(());
6476                }
6477                continue;
6478            }
6479
6480            // Nothing for the fixer to act on (`blocking == 0`) and the only
6481            // reason this round is not clean is that magi itself never got
6482            // a command to run — the shared build cache, not the patch (see
6483            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6484            // the fixer would invite a change to appease contention that has
6485            // nothing to do with the diff, and would leave this attempt
6486            // sitting in the next round's prompt as if it were about an
6487            // earlier, superseded commit rather than what it actually is:
6488            // the same head, still waiting to be checked. Wait for it the
6489            // same way the final round's own contention is already handled,
6490            // whatever round this happens to be.
6491            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6492                self.state.reviews.push(round_record);
6493                return self
6494                    .stop_reviewing(
6495                        "the round's own verification could not run",
6496                        &shell,
6497                        &winner.worktree,
6498                    )
6499                    .await;
6500            }
6501
6502            if round == max_rounds {
6503                self.state.reviews.push(round_record);
6504                return self
6505                    .stop_reviewing(
6506                        &format!(
6507                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6508                        ),
6509                        &shell,
6510                        &winner.worktree,
6511                    )
6512                    .await;
6513            }
6514
6515            // Fix. The winner's own implementer seat continues its conversation:
6516            // the competition is over, so context is pure benefit now.
6517            let blocking_findings: Vec<_> = all_findings
6518                .iter()
6519                .filter(|f| f.severity.blocks())
6520                .cloned()
6521                .collect();
6522            let fix_prompt = prompt::fix_in(
6523                &self.state.instruction,
6524                &blocking_findings,
6525                this_round_verification.as_ref(),
6526                round,
6527                max_rounds,
6528                &language,
6529                &self.state.config.graph.github_language,
6530            );
6531            let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6532            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6533            let (job, seat, out) = self
6534                .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6535                    prompt: fix_prompt.clone(),
6536                    spec,
6537                    seat,
6538                    cwd: winner.worktree.clone(),
6539                    timeout,
6540                    allow_write: true,
6541                    sessions,
6542                    artifacts: artifacts.clone(),
6543                    stem: format!("fix-{round}"),
6544                    handover: None,
6545                })
6546                .await;
6547            // Not asked: the round is not recorded (so a resume repeats it),
6548            // no fixer was handed over to, and the seat is left as it was.
6549            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6550                return self.stop_for_resource();
6551            }
6552            let agent_id = seat.agent.clone();
6553
6554            let mut fix = FixRecord {
6555                agent: agent_id,
6556                addressed: Vec::new(),
6557                rejected: Vec::new(),
6558                notes: String::new(),
6559                committed: false,
6560                failed: None,
6561                duration_ms: 0,
6562                continuation: None,
6563            };
6564            let mut continuation = ContinuationRecord::not_needed();
6565            let mut final_seat = seat.clone();
6566            match out {
6567                AgentOutcome::Ok(o) => {
6568                    fix.duration_ms = o.duration_ms;
6569                    let parsed = verdict::extract_json::<FixReport>(&o.text);
6570                    // A parsed report standing next to a command this same
6571                    // reply's own CLI never confirmed the exit status of is
6572                    // not a resolved answer — the identical `CommandEvidence`
6573                    // `state.jobs` renders, read here instead of only on
6574                    // display, per the completion judgment and the shown
6575                    // record needing to agree.
6576                    let incomplete_reason = match &parsed {
6577                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6578                            "the reply parsed, but it reported a command whose own CLI never \
6579                             confirmed an exit status"
6580                                .to_owned(),
6581                        ),
6582                        Ok(_) => None,
6583                        Err(e) => Some(e.to_string()),
6584                    };
6585                    match incomplete_reason {
6586                        None => {
6587                            let report = parsed.expect("checked Ok above");
6588                            fix.addressed = report.addressed;
6589                            fix.rejected = report.rejected;
6590                            fix.notes =
6591                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
6592                        }
6593                        Some(reason) => {
6594                            let (resumed_seat, resolved, failure, cont) = self
6595                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6596                                .await;
6597                            fix.duration_ms += cont.cumulative_wait_ms;
6598                            continuation = cont;
6599                            final_seat = resumed_seat;
6600                            match resolved {
6601                                Some(report) => {
6602                                    fix.addressed = report.addressed;
6603                                    fix.rejected = report.rejected;
6604                                    fix.notes = blind::sanitize_prose(
6605                                        &report.notes,
6606                                        &self.state.config.blind,
6607                                    );
6608                                }
6609                                None => fix.failed = failure,
6610                            }
6611                        }
6612                    }
6613                }
6614                // The CLI's raw error JSON is not a fix report to parse.
6615                AgentOutcome::Dropped(o) => {
6616                    fix.duration_ms = o.duration_ms;
6617                    let why = o
6618                        .dropped
6619                        .as_ref()
6620                        .map(|d| d.why.as_str())
6621                        .unwrap_or("the CLI ended the stream without delivering its answer");
6622                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6623                }
6624                AgentOutcome::Quota(o) => {
6625                    self.state.quota.push(QuotaLoss {
6626                        seat: final_seat.key.clone(),
6627                        node: "fix".to_owned(),
6628                        at: Timestamp::now(),
6629                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6630                    });
6631                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6632                }
6633                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6634            }
6635            fix.continuation = Some(continuation);
6636            self.state.seats.insert(final_seat.key.clone(), final_seat);
6637            let coordination = self.coordinate(&winner.worktree).await?;
6638            if let Ok(r) = git::rescue_commit(
6639                &winner.worktree,
6640                &format!("magi: review round {round} fixes (uncommitted work)"),
6641            )
6642            .await
6643            {
6644                self.state.note_withheld("fix", &r.withheld);
6645            }
6646            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6647            fix.committed = after != before;
6648            // Judged by what `git` says moved against base, never by the
6649            // fixer's own `addressed`/`rejected` count — see
6650            // `ReviewRound::progressed`. Propagated with `?`, the same as the
6651            // `patch` snapshot above: swallowing this error would default
6652            // `diff_after` to empty, which almost always differs from a
6653            // non-empty `patch` and reads as "progressed" — exactly backwards
6654            // for a `git` failure the stagnation check cannot see through.
6655            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6656            drop(coordination);
6657            let progressed = diff_after != patch;
6658            let commit_note = if fix.committed {
6659                "committed"
6660            } else {
6661                "NO new commit"
6662            };
6663            let tree_note = if progressed {
6664                "changed vs base"
6665            } else {
6666                "unchanged vs base"
6667            };
6668            self.state.event(
6669                "fix",
6670                match &fix.failed {
6671                    // Distinct on purpose from "0 addressed, 0 rejected": the
6672                    // fixer's own diff still landed (blocking counts do keep
6673                    // falling round over round), only its adoption report did
6674                    // not come back, so this must never read like every
6675                    // finding was reviewed and declined.
6676                    Some(reason) => {
6677                        format!(
6678                            "round {round}: fixer's adoption report was lost ({reason}); \
6679                             {commit_note}, tree {tree_note}"
6680                        )
6681                    }
6682                    None => format!(
6683                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6684                         {tree_note}{}",
6685                        fix.addressed.len(),
6686                        fix.rejected.len(),
6687                        if continuation.outcome == ContinuationOutcome::Resumed {
6688                            format!(
6689                                " (adoption report recovered after {} continuation(s))",
6690                                continuation.attempts
6691                            )
6692                        } else {
6693                            String::new()
6694                        },
6695                    ),
6696                },
6697            );
6698            round_record.fix = Some(fix);
6699            round_record.progressed = progressed;
6700            self.state.reviews.push(round_record);
6701            self.state.save()?;
6702
6703            // The fixer's own report never came back this round, even after
6704            // `continue_fix_report`'s own budget was spent on it — not an
6705            // ordinary "no report" (dropped stream, quota, plain failure),
6706            // which already reads that way and is left to the existing round
6707            // budget. Stopping here, rather than opening another round, is
6708            // what keeps a next reviewer/fixer wave from ever being
6709            // dispatched onto `winner.worktree` while whatever the seat's
6710            // last call may still have running there is unaccounted for: no
6711            // process liveness check exists (and none is being added — see
6712            // AGENTS.md/this task's own scope), so the only way to honour
6713            // "nothing starts before a valid report returns" is to not start
6714            // anything further on this worktree from this run at all.
6715            if matches!(
6716                continuation.outcome,
6717                ContinuationOutcome::Exhausted
6718                    | ContinuationOutcome::QuotaLost
6719                    | ContinuationOutcome::NoSession
6720            ) {
6721                return self
6722                    .stop_reviewing(
6723                        "the fixer's adoption report never came back, even after resuming its \
6724                         own seat; refusing to start another round against the same worktree \
6725                         while that is unresolved",
6726                        &shell,
6727                        &winner.worktree,
6728                    )
6729                    .await;
6730            }
6731
6732            let streak = self
6733                .state
6734                .reviews
6735                .iter()
6736                .rev()
6737                .take_while(|r| !r.progressed)
6738                .count();
6739            if streak >= STAGNANT_LIMIT {
6740                return self
6741                    .stop_reviewing(
6742                        &format!(
6743                            "the tree has not moved against base for {streak} round(s) in a row"
6744                        ),
6745                        &shell,
6746                        &winner.worktree,
6747                    )
6748                    .await;
6749            }
6750        }
6751        Ok(())
6752    }
6753
6754    /// Decide, from the last recorded round's own verification, whether
6755    /// stopping the review loop is a hand-off or a genuine block.
6756    ///
6757    /// Called once the loop has given up trying — the round budget is spent,
6758    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
6759    /// findings still open, never while a round is still clean or the
6760    /// incomplete-panel case handled inline above. Gate and e2e are facts
6761    /// about the tree; a lingering review finding is an opinion, and this
6762    /// workload's own `magi stats` puts reviewer precision low enough
6763    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
6764    /// must not by itself stand between a green, verified change and the
6765    /// human who decides what to do with it. A red e2e is not an opinion, so
6766    /// that case still blocks, with the failing command and a tail of its
6767    /// output recorded here rather than left in `run.json` for someone to go
6768    /// find.
6769    ///
6770    /// A round that deferred its own e2e (see [`Config::graph`]'s
6771    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
6772    /// only because nothing ran, and treating an empty list as a passing one
6773    /// here is exactly the "deferred painted green" bug this function exists
6774    /// to not have. When the last round's own verification never resolved —
6775    /// deferred on purpose, or a real attempt the shared build cache blocked
6776    /// — this makes (or retries) the real run, on the actual worktree this
6777    /// loop is about to stop touching, before deciding anything. A
6778    /// resource-blocked attempt is likewise never read as either green or
6779    /// red: it is evidence about the machine, not the patch (see
6780    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
6781    /// blocked cache leaves this call without deciding rather than guessing
6782    /// — the caller retries on a later reentry.
6783    /// Record, once, that the review loop handed off over a blocking finding
6784    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
6785    /// `land` asks the owner even with `land_approval` off. Called from every
6786    /// path that concludes `Gating`; a reentry keeps the first record.
6787    fn record_contested_handoff(&mut self) {
6788        if self.state.contested_handoff.is_some() {
6789            return;
6790        }
6791        let Some(contested) = self
6792            .state
6793            .reviews
6794            .last()
6795            .and_then(ReviewRound::contested_handoff)
6796        else {
6797            return;
6798        };
6799        self.state.event(
6800            "review",
6801            format!(
6802                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6803                 wait for the owner's approval",
6804                contested.findings.len(),
6805                contested.rejecters.len()
6806            ),
6807        );
6808        self.state.contested_handoff = Some(contested);
6809    }
6810
6811    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6812        let round_idx = self.state.reviews.len() - 1;
6813        // A deferred round and a resource-blocked one are the same shape
6814        // here: neither has a real result yet, and both get one more
6815        // attempt. Read off `e2e_status` — the single source for this —
6816        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
6817        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
6818        // still retries instead of being read as a settled result the
6819        // instant it stops being empty.
6820        let needs_catchup_run = matches!(
6821            self.state.reviews[round_idx].e2e_status(),
6822            E2eStatus::Deferred | E2eStatus::ResourceBlocked
6823        );
6824        if needs_catchup_run {
6825            let round = self.state.reviews[round_idx].round;
6826            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6827            let commands = self.state.config.verify.e2e.clone();
6828            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6829            let cache_dir = self.state.config.cache_dir();
6830            let context = format!(
6831                "round {round}: verification unresolved, catching up before the final decision"
6832            );
6833            let wait = self.repository_wait();
6834            let (outcomes, verify_retried) = with_cache_lease(
6835                &mut self.state,
6836                cache_dir.as_deref(),
6837                "e2e",
6838                "e2e",
6839                worktree,
6840                &attempted_head,
6841                timeout,
6842                &context,
6843                &wait,
6844                |state, budget| {
6845                    let shell = shell.to_vec();
6846                    let commands = commands.clone();
6847                    let context = context.clone();
6848                    async move {
6849                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6850                            .await
6851                    }
6852                },
6853            )
6854            .await?;
6855            let last = &mut self.state.reviews[round_idx];
6856            last.e2e = outcomes;
6857            last.verify_retried = verify_retried;
6858            // Always the commit and time this attempt actually targeted,
6859            // whether or not it happens to equal the reviewed `head` and
6860            // whether or not a command finished — see
6861            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
6862            // attempt is recorded too, so a later reader sees "attempted
6863            // again at T2" rather than silence.
6864            last.verified_head = Some(attempted_head);
6865            last.verified_at = Some(Timestamp::now());
6866            if verify_inconclusive(&last.e2e) {
6867                // Still not a real result: `e2e_deferred` is left exactly
6868                // as it was, so `needs_catchup_run` above reads
6869                // `ResourceBlocked` (via `e2e_status`, which checks
6870                // `resource_blocked` before `e2e_deferred`) and retries
6871                // again on the next reentry, rather than recording
6872                // contention as a red e2e and blocking the run on it.
6873                self.state.save()?;
6874                return Ok(());
6875            }
6876            last.e2e_deferred = false;
6877        }
6878        let last = &self.state.reviews[round_idx];
6879        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6880
6881        match last.e2e_status() {
6882            E2eStatus::Failed => {
6883                let red: Vec<String> = last
6884                    .e2e
6885                    .iter()
6886                    .filter(|o| !o.ok())
6887                    .map(|o| {
6888                        format!(
6889                            "`{}` -> {:?}\n{}",
6890                            o.command,
6891                            o.code,
6892                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6893                        )
6894                    })
6895                    .collect();
6896                self.state
6897                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6898                self.state.status = RunStatus::Blocked;
6899            }
6900            // `needs_catchup_run` above already retried once this call; if
6901            // it is still blocked, this is magi's own admission it could
6902            // not get a command to run, never a verdict on the patch — the
6903            // run is left exactly where a later reentry can retry again.
6904            E2eStatus::ResourceBlocked => {
6905                self.state.event(
6906                    "review",
6907                    format!(
6908                        "{why}; e2e could not run (shared build cache unavailable); not \
6909                         deciding yet"
6910                    ),
6911                );
6912            }
6913            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6914                self.state.event(
6915                    "review",
6916                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6917                );
6918                self.record_contested_handoff();
6919                self.state.status = RunStatus::Gating;
6920            }
6921        }
6922        self.state.save()?;
6923        Ok(())
6924    }
6925
6926    // ----------------------------------------------------------------- gate
6927
6928    async fn gate(&mut self) -> Result<()> {
6929        // Judged by the review record itself, not by `status`: a solo
6930        // candidate's `judge`/`deliberate` skip rewrites `status` on every
6931        // reentry (see `judge`), and trusting it here is exactly how a run
6932        // that exhausted its review budget got gated and merged a second
6933        // time around. `review_conclusion` recomputes the review loop's own
6934        // verdict from the round records themselves — `Gating` for a clean
6935        // round or a hand-off (see `stop_reviewing`), anything else means the
6936        // loop is still going or genuinely blocked.
6937        // A base the winner could not be replayed onto is a decision, not a
6938        // round: there is no landing tree to gate. Read as its own record for
6939        // the same reason the review verdict is.
6940        if self.state.status == RunStatus::Failed
6941            || self
6942                .state
6943                .base_sync
6944                .as_ref()
6945                .is_some_and(|s| s.conflict.is_some())
6946            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6947                != Some(RunStatus::Gating)
6948        {
6949            return Ok(());
6950        }
6951        if self.state.gate_ran {
6952            // `review_loop` derives its conclusion from the clean review
6953            // record on every reentry and therefore puts a completed run back
6954            // in `Gating`. A recorded gate is a stronger, terminal fact:
6955            // retain its original command output (or lack of any, for a repo
6956            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
6957            // doc) and restore `Blocked` on a real failure rather than
6958            // pretending the command is still running or running it a second
6959            // time. `gate_ran == false` remains the only shape — unattempted,
6960            // or a resource-blocked retry — that may still need to execute a
6961            // command.
6962            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6963                self.state.status = RunStatus::Blocked;
6964                self.state.save()?;
6965            }
6966            return Ok(());
6967        }
6968        let Some(winner) = self.state.winner().cloned() else {
6969            return Ok(());
6970        };
6971        self.state.status = RunStatus::Gating;
6972        let mut outcomes = self.run_gate(&winner).await?;
6973        loop {
6974            // A resource-blocked outcome means the gate command never actually
6975            // ran - the shared build cache could not be acquired or confirmed
6976            // fresh in time - which is evidence about the machine, not about
6977            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
6978            // Recording it as a red gate would mark a run `Blocked` on nothing
6979            // but contention magi has already logged; leaving `self.state.gate`
6980            // empty and `self.state.gate_ran` false instead keeps the shape
6981            // this function already treats as "still needs to run" (see the
6982            // early-return above), so the next call retries the command
6983            // rather than concluding anything.
6984            if verify_inconclusive(&outcomes) {
6985                self.state.save()?;
6986                return Ok(());
6987            }
6988            if outcomes.iter().all(CommandOutcome::ok) {
6989                break;
6990            }
6991            match self.gate_fix_round(&winner, &outcomes).await? {
6992                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6993                GateFix::Stop => break,
6994                GateFix::Defer => {
6995                    self.state.save()?;
6996                    return Ok(());
6997                }
6998            }
6999        }
7000        let passed = outcomes.iter().all(CommandOutcome::ok);
7001        self.state.gate = outcomes;
7002        self.state.gate_ran = true;
7003        self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7004        if !passed {
7005            self.state.status = RunStatus::Blocked;
7006            let spent = self.state.gate_fixes.len();
7007            self.state.event(
7008                "gate",
7009                if spent == 0 {
7010                    "gate failed; not merging".to_owned()
7011                } else {
7012                    format!("gate failed after {spent} gate-fix round(s); not merging")
7013                },
7014            );
7015        }
7016        self.state.save()?;
7017        Ok(())
7018    }
7019
7020    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7021    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7022    /// after review is clean and never on a candidate awaiting judging.
7023    ///
7024    /// Never fails the run: a non-zero exit or timeout is a warning and a
7025    /// recorded outcome, and the gate remains the single arbiter. Nothing
7026    /// configured means nothing happens - no event, no commit. `commit_all`
7027    /// commits any leftover change under the neutral identity and returns
7028    /// `false` when the tree is clean, so no empty commit is ever made.
7029    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7030        let commands = state.config.verify.pre_gate.clone();
7031        if commands.is_empty() {
7032            return;
7033        }
7034        let shell = state.config.shell();
7035        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7036        let (outcomes, _) = run_commands(
7037            state,
7038            "pre_gate",
7039            "pre_gate",
7040            0,
7041            &shell,
7042            &commands,
7043            &winner.worktree,
7044            timeout,
7045        )
7046        .await;
7047        for o in &outcomes {
7048            if !o.ok() {
7049                tracing::warn!(
7050                    "pre_gate `{}` failed ({:?}); the gate decides",
7051                    o.command,
7052                    o.code
7053                );
7054            }
7055            state.event(
7056                "pre_gate",
7057                format!(
7058                    "`{}` -> {}",
7059                    o.command,
7060                    if o.ok() {
7061                        "pass".to_owned()
7062                    } else {
7063                        format!(
7064                            "FAIL ({:?})\n{}",
7065                            o.code,
7066                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7067                        )
7068                    }
7069                ),
7070            );
7071        }
7072        state.pre_gate = outcomes;
7073        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7074            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7075                Ok(head) => {
7076                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7077                    state.pre_gate_commit = Some(head);
7078                }
7079                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7080            },
7081            Ok(false) => {}
7082            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7083        }
7084        if let Err(e) = state.save() {
7085            tracing::warn!("could not persist the pre_gate record: {e:#}");
7086        }
7087    }
7088
7089    /// Run `verify.gate` once against the winner's current tree, logging one
7090    /// event per command. Empty when nothing is configured.
7091    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7092        let wait = self.repository_wait();
7093        run_landing_gate(&mut self.state, winner, &wait).await
7094    }
7095
7096    /// One bounded fix round for a failing gate.
7097    ///
7098    /// The fixer is told the failure came from the gate itself, not from a
7099    /// reviewer, and is shown the failed commands, their exit codes and a tail
7100    /// of their output - whatever `[verify].gate` holds, nothing here knows
7101    /// what those commands run. Only a normal non-zero exit that printed
7102    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7103    /// command or a full disk says nothing about the code, and a fixer sent
7104    /// after it can only appease the machine. The round is judged by what git
7105    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7106    /// again before the gate does, so a fix cannot trade a green gate for a
7107    /// red e2e unnoticed.
7108    async fn gate_fix_round(
7109        &mut self,
7110        winner: &Candidate,
7111        outcomes: &[CommandOutcome],
7112    ) -> Result<GateFix> {
7113        let cap = self.state.config.graph.gate_fix_rounds;
7114        let spent = self.state.gate_fixes.len();
7115        if spent >= cap {
7116            if cap > 0 {
7117                self.state.event(
7118                    "gate",
7119                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7120                );
7121            }
7122            return Ok(GateFix::Stop);
7123        }
7124        if !gate_fixable(outcomes) {
7125            self.state.event(
7126                "gate",
7127                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7128                 command or similar); not spending a fix round on it",
7129            );
7130            return Ok(GateFix::Stop);
7131        }
7132        let min_free = self.state.config.disk.min_free_bytes;
7133        if min_free > 0 {
7134            match crate::disk::free_bytes(&winner.worktree) {
7135                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7136                Ok(free) => {
7137                    self.state.event(
7138                        "gate",
7139                        format!(
7140                            "only {free} bytes free ({min_free} required by `[disk] \
7141                             min_free_bytes`); not spending a fix round on a failure the disk \
7142                             may explain"
7143                        ),
7144                    );
7145                    return Ok(GateFix::Stop);
7146                }
7147                Err(e) => {
7148                    self.state.event(
7149                        "gate",
7150                        format!("free disk space could not be measured ({e:#}); no fix round"),
7151                    );
7152                    return Ok(GateFix::Stop);
7153                }
7154            }
7155        }
7156
7157        let attempt = spent + 1;
7158        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7159        let base = self.landing_base();
7160        let fix_prompt = prompt::gate_fix_in(
7161            &self.state.instruction,
7162            &failed,
7163            attempt,
7164            cap,
7165            &self.state.config.graph.language,
7166            &self.state.config.graph.github_language,
7167        );
7168        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7169        let sessions = self.state.config.graph.sessions;
7170        let artifacts = self.merge_repair_artifacts();
7171        self.state.event(
7172            "gate",
7173            format!("gate failed; gate-fix round {attempt} of {cap}"),
7174        );
7175        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7176        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7177        // Persist before invoking the fixer: it may commit and then be
7178        // interrupted before its result (or the following e2e) is saved.
7179        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7180            head: before.clone(),
7181            passed: false,
7182            outcomes: Vec::new(),
7183        });
7184        self.state.save()?;
7185        let (_, seat, out) = self
7186            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7187                prompt: fix_prompt.clone(),
7188                spec,
7189                seat,
7190                cwd: winner.worktree.clone(),
7191                timeout,
7192                allow_write: true,
7193                sessions,
7194                artifacts: artifacts.clone(),
7195                stem: format!("gate-fix-{attempt}"),
7196                handover: None,
7197            })
7198            .await;
7199        // The fixer was never asked: no round is spent (nothing is recorded)
7200        // and a later reentry asks the same agent again.
7201        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7202            return Ok(GateFix::Defer);
7203        }
7204        let mut record = GateFixRecord {
7205            agent: seat.agent.clone(),
7206            failed,
7207            notes: String::new(),
7208            committed: false,
7209            error: None,
7210        };
7211        match out {
7212            AgentOutcome::Ok(o) => {
7213                // A missing report is not a failed fix: the round is judged
7214                // by the tree below, and the report only carries prose.
7215                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7216                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7217                }
7218            }
7219            AgentOutcome::Dropped(_) => {
7220                record.error = Some("the CLI dropped the stream".to_owned());
7221            }
7222            AgentOutcome::Quota(o) => {
7223                self.state.quota.push(QuotaLoss {
7224                    seat: seat.key.clone(),
7225                    node: "gate-fix".to_owned(),
7226                    at: Timestamp::now(),
7227                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7228                });
7229                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7230            }
7231            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7232        }
7233        self.state.seats.insert(seat.key.clone(), seat);
7234        let coordination = self.coordinate(&winner.worktree).await?;
7235        if let Ok(r) = git::rescue_commit(
7236            &winner.worktree,
7237            &format!("magi: gate fix {attempt} (uncommitted work)"),
7238        )
7239        .await
7240        {
7241            self.state.note_withheld("gate-fix", &r.withheld);
7242        }
7243        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7244        record.committed = after != before;
7245        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7246        let note = record.error.clone();
7247        self.state.gate_fixes.push(record);
7248        self.state.save()?;
7249        if !changed {
7250            self.state.event(
7251                "gate",
7252                match note {
7253                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7254                    None => format!("gate-fix round {attempt}: the tree did not change"),
7255                },
7256            );
7257            return Ok(GateFix::Stop);
7258        }
7259        self.state.event(
7260            "gate",
7261            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7262        );
7263
7264        drop(coordination);
7265        let wait = self.repository_wait();
7266        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7267    }
7268
7269    // ---------------------------------------------------------------- merge
7270
7271    /// One read-only rewrite by the summary's author; a text the gate still
7272    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7273    /// applies on their word or on silence. `None` means the run is parked on
7274    /// that question and nothing may be pushed or opened yet.
7275    async fn guarded_pr_message(
7276        &mut self,
7277        winner: &Candidate,
7278        facts: Option<&BranchFacts>,
7279        posting: bool,
7280    ) -> Result<Option<PrMessage>> {
7281        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7282        let head = match final_head.as_deref() {
7283            Some(h) => crate::pr_outcome::FinalHead::At(h),
7284            None => crate::pr_outcome::FinalHead::Unknown,
7285        };
7286        // Only a pull request that is about to be posted pays for a summary.
7287        let summarized = if posting {
7288            Some(self.summarize_change(winner, final_head.as_deref()).await)
7289        } else {
7290            None
7291        };
7292        // A busy shared resource is not a permanent cause: publishing an
7293        // "unavailable" description would stick (a later head move never
7294        // edits the body). Stop before anything is pushed or opened; the
7295        // run is `Blocked` and a resume asks the summary again.
7296        if self.state.resource_block.is_some() {
7297            self.state.event(
7298                "pr-summary",
7299                "a shared resource was busy; nothing was published, resume to summarize again",
7300            );
7301            return Ok(None);
7302        }
7303        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7304            anyhow::bail!(
7305                "the final commit moved during public description generation; retry before publishing"
7306            );
7307        }
7308        let note = match (&summarized, &self.state.change_summary) {
7309            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7310            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7311            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7312            (None, _) => ChangeNote::NotAsked,
7313        };
7314        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7315        if !posting {
7316            let identity = crate::scrub::Identity::current();
7317            return Ok(Some(PrMessage {
7318                title: crate::scrub::scrub(&pr.title, &identity),
7319                body: crate::scrub::scrub(&pr.body, &identity),
7320            }));
7321        }
7322        // This very text was already rejected and asked about: no second
7323        // judge call, rewrite or question, just carry the record on.
7324        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7325        if let Some(known) = self
7326            .state
7327            .github_text
7328            .clone()
7329            .filter(|g| g.fingerprint == raw_fp)
7330        {
7331            return self.settle_withheld(winner, &pr, known).await;
7332        }
7333        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7334        let mut violations =
7335            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7336        let identity = crate::scrub::Identity::current();
7337        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7338            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7339            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7340        {
7341            violations.push(crate::github_text::Violation::SensitiveData);
7342        }
7343        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7344        // only a language violation starts one; sensitive data alone goes
7345        // straight to the owner, with the redacted text as the fallback.
7346        let has_language = violations
7347            .iter()
7348            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7349        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7350            self.state.event(
7351                "github-text",
7352                if has_language {
7353                    format!("description rejected: {violations:?}; requesting one rewrite")
7354                } else {
7355                    format!("description rejected: {violations:?}; asking the owner")
7356                },
7357            );
7358            let mut rewritten = false;
7359            let mut rejected_rewrite = None;
7360            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7361            {
7362                let output = text;
7363                if let Some(title) = summary_title(&output) {
7364                    // Judge the redacted candidate that will actually be posted.
7365                    // Sensitive data alone must not discard a useful rewrite.
7366                    let raw_body = summary_without_title(&output);
7367                    let clean_title = crate::scrub::scrub(&title, &identity);
7368                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7369                    let redacted = clean_title != title || body != raw_body;
7370                    let title = clean_title;
7371                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7372                    if !complete {
7373                        self.state.event(
7374                            "github-text",
7375                            "description rewrite omitted required findings",
7376                        );
7377                    }
7378                    if !body.trim().is_empty() {
7379                        body.push_str("\n\n");
7380                        body.push_str(&public_pr_evidence(&self.state, head));
7381                    }
7382                    let marker = format!("magi:run/{}", self.state.id);
7383                    if !body.trim().is_empty() && !body.contains(&marker) {
7384                        body.push_str(&format!("\n\n{marker}"));
7385                    }
7386                    let rewrite = PrMessage {
7387                        title: title.clone(),
7388                        body: body.clone(),
7389                    };
7390                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7391                    let mut failed = crate::github_text::check_for(
7392                        &self.state.config.graph,
7393                        &title,
7394                        &body,
7395                        redecision,
7396                    );
7397                    if (!crate::github_text::shareable(&title)
7398                        || !crate::github_text::shareable(&body))
7399                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7400                    {
7401                        failed.push(crate::github_text::Violation::SensitiveData);
7402                    }
7403                    if complete
7404                        && !title.trim().is_empty()
7405                        && !body.trim().is_empty()
7406                        && !failed.is_empty()
7407                    {
7408                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7409                            title: title.clone(),
7410                            body: body.clone(),
7411                            failed: failed
7412                                .iter()
7413                                .map(|v| crate::github_text::category(*v).to_owned())
7414                                .collect(),
7415                        });
7416                    }
7417                    if complete
7418                        && !title.trim().is_empty()
7419                        && !raw_body.trim().is_empty()
7420                        && failed.is_empty()
7421                    {
7422                        decision = redecision;
7423                        pr = PrMessage { title, body };
7424                        rewritten = true;
7425                        self.state
7426                            .event("github-text", "description rewrite passed");
7427                        if redacted {
7428                            self.state.event(
7429                                "github-text",
7430                                "sensitive data removed from description rewrite",
7431                            );
7432                        }
7433                    }
7434                }
7435            }
7436            if !rewritten {
7437                self.state.event(
7438                    "github-text",
7439                    "description rewrite unavailable or rejected; asking the owner",
7440                );
7441                if let Some(w) =
7442                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7443                {
7444                    // A different text than the one asked about before: that
7445                    // question no longer describes anything.
7446                    self.retire_github_text_question("the withheld text changed");
7447                    let record = crate::run::GithubTextAsk {
7448                        rewrite: rejected_rewrite,
7449                        fingerprint: raw_fp,
7450                        title: w.title,
7451                        body: w.body,
7452                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7453                        question: None,
7454                        asks: 0,
7455                        resolved: false,
7456                        chosen_title: None,
7457                        choice: None,
7458                    };
7459                    return self.settle_withheld(winner, &pr, record).await;
7460                }
7461            }
7462        }
7463        let (title, body) =
7464            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7465        Ok(Some(PrMessage { title, body }))
7466    }
7467
7468    /// Describe the whole change from its final diff in either run mode.
7469    ///
7470    /// The summary is tied to the final commit and to the merge base of the
7471    /// resolved remote base; a record for any other pair is dropped and the
7472    /// agent asked again, so a description never outlives the diff it was
7473    /// written from. The seat is read-only and the prompt tells it to read the
7474    /// diff and the code around it. `[roles] synthesizer` selects the agents
7475    /// (unset: the default pick), each asked at most once, advancing on an
7476    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7477    /// carries the reason it is unavailable; nothing is invented then.
7478    async fn summarize_change(
7479        &mut self,
7480        winner: &Candidate,
7481        head: Option<&str>,
7482    ) -> std::result::Result<(), String> {
7483        let Some(head) = head else {
7484            return Err("the final commit could not be read".to_owned());
7485        };
7486        let remote = self.state.config.merge.remote.clone();
7487        let tracking = format!("{remote}/{}", self.state.base_branch);
7488        if !git::rev_exists(&winner.worktree, &tracking).await {
7489            return Err(format!("the remote base `{tracking}` could not be read"));
7490        }
7491        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7492            Ok(b) if !b.is_empty() => b,
7493            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7494        };
7495        let evidence = crate::pr_outcome::summary_evidence(
7496            &self.state,
7497            crate::pr_outcome::FinalHead::At(head),
7498        );
7499        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7500        if self
7501            .state
7502            .change_summary
7503            .as_ref()
7504            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7505        {
7506            return Ok(());
7507        }
7508        if self.state.change_summary.take().is_some() {
7509            self.state.event(
7510                "pr-summary",
7511                "the final diff or evidence changed; the earlier change summary was discarded",
7512            );
7513        }
7514        // Save the source snapshot even if no generation agent is available.
7515        if let Err(e) = write_artifact(
7516            &self.state,
7517            &format!("public-description-{fingerprint}.md"),
7518            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7519        ) {
7520            tracing::warn!("could not save public description evidence: {e:#}");
7521        }
7522        let chain = match agent::pick_chain(
7523            &self.state.config.agents,
7524            self.state.config.roles.synthesizer.as_ref(),
7525            &agent::installed,
7526            "synthesizer",
7527        ) {
7528            Ok(chain) => chain,
7529            Err(e) => {
7530                tracing::warn!("no agent for the change summary: {e:#}");
7531                self.state.event("pr-summary", "no summary agent available");
7532                return Err("no agent was available to summarize it".to_owned());
7533            }
7534        };
7535        let subjects = git::subjects(&winner.worktree, &base, head)
7536            .await
7537            .unwrap_or_default();
7538        let stat = git::diff_stat(&winner.worktree, &base, head)
7539            .await
7540            .unwrap_or_default();
7541        let stat: String = stat.chars().take(6_000).collect();
7542        let mut prompt = crate::prompt::change_summary_in(
7543            &base,
7544            head,
7545            &subjects,
7546            stat.trim(),
7547            &self.state.config.graph.github_language,
7548        );
7549        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7550        let sessions = self.state.config.graph.sessions;
7551        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7552        let prompts = self.state.config.prompts.clone();
7553        let cache = self.state.config.cache_dir();
7554        let run = self.state.id.clone();
7555        for (i, spec) in chain.into_iter().enumerate() {
7556            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7557            let github_spec = match agent::for_github_text(
7558                &spec,
7559                &self.state.config.graph.github_language,
7560                &winner.worktree,
7561            ) {
7562                Ok(spec) => spec,
7563                Err(e) => {
7564                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7565                    continue;
7566                }
7567            };
7568            let job = SeatJob {
7569                spec: github_spec,
7570                seat,
7571                cwd: winner.worktree.clone(),
7572                prompt: prompt.clone(),
7573                timeout,
7574                allow_write: false,
7575                sessions,
7576                artifacts: agent::artifacts_dir(&self.state.dir()),
7577                stem: if i == 0 {
7578                    "change-summary".to_owned()
7579                } else {
7580                    format!("change-summary-{}", spec.id)
7581                },
7582                handover: None,
7583            };
7584            let ctx = WaveCtx {
7585                run: &run,
7586                node: "pr-summary",
7587                prompts: &prompts,
7588                cache: cache.as_deref(),
7589                round: None,
7590                carry_seats: false,
7591                pauses: Some((&self.pause, &self.interrupt)),
7592            };
7593            let (seat, outcome) =
7594                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7595            self.state.seats.insert(seat.key.clone(), seat);
7596            let text = match outcome {
7597                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7598                AgentOutcome::ResourceBlocked(why) => {
7599                    // No agent was asked: the run stops resumably before
7600                    // anything is published (see `guarded_pr_message`).
7601                    if self.state.resource_block.is_none() {
7602                        self.state.resource_block = Some(crate::run::ResourceBlock {
7603                            node: "pr-summary".to_owned(),
7604                            kind: "shared resource".to_owned(),
7605                            detail: why,
7606                            at: Timestamp::now(),
7607                        });
7608                    }
7609                    return Err("a shared resource was busy".to_owned());
7610                }
7611                _ => continue,
7612            };
7613            let blind = &self.state.config.blind;
7614            let Some(title) = summary_title(&text) else {
7615                continue;
7616            };
7617            let body: String = summary_without_title(&text)
7618                .lines()
7619                .filter(|l| !l.contains("magi:run/"))
7620                .collect::<Vec<_>>()
7621                .join("\n");
7622            let body = blind::sanitize_prose(body.trim(), blind);
7623            let title = blind::sanitize_prose(&title, blind);
7624            if body.trim().is_empty() || title.trim().is_empty() {
7625                continue;
7626            }
7627            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7628                continue;
7629            }
7630            // The branch must still be the one the agent was shown.
7631            if git::rev_parse(&winner.worktree, "HEAD")
7632                .await
7633                .ok()
7634                .as_deref()
7635                != Some(head)
7636            {
7637                return Err("the branch moved while the summary was written".to_owned());
7638            }
7639            if git::merge_base(&winner.worktree, &tracking, head)
7640                .await
7641                .ok()
7642                .as_deref()
7643                != Some(base.as_str())
7644                || crate::pr_outcome::summary_evidence(
7645                    &self.state,
7646                    crate::pr_outcome::FinalHead::At(head),
7647                ) != evidence
7648            {
7649                return Err("the summary evidence changed while it was written".to_owned());
7650            }
7651            self.state.change_summary = Some(crate::run::ChangeSummary {
7652                head: head.to_owned(),
7653                base,
7654                evidence: fingerprint,
7655                title,
7656                body,
7657            });
7658            self.state.event(
7659                "pr-summary",
7660                format!("change summary written by {} for the final diff", spec.id),
7661            );
7662            if let Err(e) = self.state.save() {
7663                tracing::warn!("could not save the change summary: {e:#}");
7664            }
7665            return Ok(());
7666        }
7667        self.state
7668            .event("pr-summary", "no agent produced a change summary");
7669        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7670    }
7671
7672    /// One read-only rewrite request for a withheld description. The
7673    /// implementer's own conversation is tried first when it exists and can be
7674    /// resumed; then the `[roles] language_judge` chain (or the default pick)
7675    /// writes it in a separate conversation, skipping the implementer's agent.
7676    /// The prompt carries the whole text, so no agent needs any memory. Each
7677    /// agent is asked at most once and the chain advances on an error, quota,
7678    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
7679    /// that has run out of quota must not end the rewrite. Nothing is recorded
7680    /// as a handover.
7681    async fn rewrite_pr_text(
7682        &mut self,
7683        winner: &Candidate,
7684        pr: &PrMessage,
7685        violations: &[crate::github_text::Violation],
7686    ) -> Option<String> {
7687        let prompt = format!(
7688            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7689            pr.title, pr.body
7690        );
7691        let sessions = self.state.config.graph.sessions;
7692        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7693        // (spec, the resumed implementer's seat). A fallback seat is created
7694        // only when its turn comes, so an unused fallback leaves no record.
7695        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
7696        let key = format!("impl-{}", winner.label);
7697        if let Some(spec) = self
7698            .state
7699            .config
7700            .agents
7701            .iter()
7702            .find(|a| a.id == winner.agent)
7703            .cloned()
7704            && let Some(existing) = self.state.seats.get(&key)
7705            && existing.agent == spec.id
7706            && agent::has_session(spec.kind, existing, sessions)
7707        {
7708            attempts.push((spec, Some(existing.clone())));
7709        }
7710        match agent::pick_chain(
7711            &self.state.config.agents,
7712            self.state.config.roles.language_judge.as_ref(),
7713            &agent::installed,
7714            "language_judge",
7715        ) {
7716            Ok(chain) => {
7717                for spec in chain {
7718                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
7719                        continue;
7720                    }
7721                    attempts.push((spec, None));
7722                }
7723            }
7724            Err(e) if attempts.is_empty() => {
7725                tracing::warn!("no agent for the description rewrite: {e:#}");
7726                self.state
7727                    .event("github-text", "no rewrite agent available");
7728                return None;
7729            }
7730            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
7731        }
7732        let prompts = self.state.config.prompts.clone();
7733        let cache = self.state.config.cache_dir();
7734        let run = self.state.id.clone();
7735        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
7736            let resumed = resumed_seat.is_some();
7737            let seat = resumed_seat.unwrap_or_else(|| {
7738                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
7739            });
7740            let github_spec = match agent::for_github_text(
7741                &spec,
7742                &self.state.config.graph.github_language,
7743                &winner.worktree,
7744            ) {
7745                Ok(spec) => spec,
7746                Err(e) => {
7747                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7748                    continue;
7749                }
7750            };
7751            let job = SeatJob {
7752                spec: github_spec,
7753                seat,
7754                cwd: winner.worktree.clone(),
7755                prompt: prompt.clone(),
7756                timeout: retry_budget(timeout_full, resumed),
7757                allow_write: false,
7758                sessions,
7759                artifacts: agent::artifacts_dir(&self.state.dir()),
7760                stem: if i == 0 {
7761                    "github-text-rewrite".to_owned()
7762                } else {
7763                    format!("github-text-rewrite-{}", spec.id)
7764                },
7765                handover: None,
7766            };
7767            let ctx = WaveCtx {
7768                run: &run,
7769                node: "github-text",
7770                prompts: &prompts,
7771                cache: cache.as_deref(),
7772                round: None,
7773                carry_seats: false,
7774                pauses: Some((&self.pause, &self.interrupt)),
7775            };
7776            let (seat, outcome) =
7777                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7778            self.state.seats.insert(seat.key.clone(), seat);
7779            match outcome {
7780                AgentOutcome::Ok(o)
7781                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7782                {
7783                    return Some(o.text);
7784                }
7785                AgentOutcome::ResourceBlocked(_) => return None,
7786                _ => {}
7787            }
7788        }
7789        None
7790    }
7791
7792    /// Abandon the standing posting-gate question, if any.
7793    fn retire_github_text_question(&mut self, why: &str) {
7794        let Some(id) = self
7795            .state
7796            .github_text
7797            .as_ref()
7798            .and_then(|g| g.question.clone())
7799        else {
7800            return;
7801        };
7802        let _ = ask::Questions::open().update(&id, |q| {
7803            q.abandon(why);
7804            Ok(())
7805        });
7806    }
7807
7808    /// Carry a withheld text to its end: file the question, park while it is
7809    /// open, and on an answer, silence or timeout pick the text to post. The
7810    /// decision is saved before it is used, so it applies once.
7811    async fn settle_withheld(
7812        &mut self,
7813        winner: &Candidate,
7814        pr: &PrMessage,
7815        mut g: crate::run::GithubTextAsk,
7816    ) -> Result<Option<PrMessage>> {
7817        use crate::github_text as gt;
7818        let store = ask::Questions::open();
7819        let timeout = self.state.config.graph.answer_timeout;
7820        if !g.resolved {
7821            let mut standing = g
7822                .question
7823                .as_ref()
7824                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7825            if let Some(q) = standing.as_ref()
7826                && q.status.open()
7827                && gt::expired(q, timeout)
7828            {
7829                let why = format!("no answer within {}s of asking", timeout.max(1));
7830                standing = store
7831                    .update(&q.id, |q| {
7832                        q.abandon(&why);
7833                        Ok(())
7834                    })
7835                    .ok()
7836                    .map(|(q, ())| q);
7837            }
7838            let mut ask_again: Option<Option<Vec<String>>> = None;
7839            match standing {
7840                // Recorded but never written (a stop between the two saves),
7841                // or removed: ask again without spending the retry.
7842                None if g.asks <= 1 => {
7843                    g.asks = 0;
7844                    ask_again = Some(None);
7845                }
7846                None => {
7847                    g.resolved = true;
7848                    g.chosen_title = None;
7849                    g.choice = Some(GithubTextChoice::Fallback);
7850                }
7851                Some(q) if q.status.open() => {
7852                    self.park_on_github_text(&g)?;
7853                    return Ok(None);
7854                }
7855                Some(q) => match gt::read_reply(&q) {
7856                    gt::Reply::Fallback => {
7857                        g.resolved = true;
7858                        g.chosen_title = None;
7859                        g.choice = Some(GithubTextChoice::Fallback);
7860                    }
7861                    gt::Reply::Original => {
7862                        g.resolved = true;
7863                        g.chosen_title = None;
7864                        g.choice = Some(GithubTextChoice::Original);
7865                    }
7866                    gt::Reply::Rewrite => {
7867                        g.resolved = true;
7868                        g.chosen_title = None;
7869                        g.choice = Some(if g.rewrite.is_some() {
7870                            GithubTextChoice::Rewrite
7871                        } else {
7872                            GithubTextChoice::Fallback
7873                        });
7874                    }
7875                    gt::Reply::Title(text) => {
7876                        let first = text
7877                            .lines()
7878                            .map(str::trim)
7879                            .find(|l| !l.is_empty())
7880                            .unwrap_or("");
7881                        let decision = if first.is_empty() {
7882                            None
7883                        } else {
7884                            let candidate = PrMessage {
7885                                title: first.to_owned(),
7886                                body: String::new(),
7887                            };
7888                            self.judge_pr_language(&winner.worktree, &candidate).await
7889                        };
7890                        match gt::vet_title(
7891                            &text,
7892                            decision,
7893                            self.state.config.graph.github_english(),
7894                        ) {
7895                            Ok(title) => {
7896                                g.resolved = true;
7897                                g.chosen_title = Some(title);
7898                                g.choice = Some(GithubTextChoice::MyTitle);
7899                            }
7900                            Err(cats) if g.asks < 2 => {
7901                                ask_again = Some(Some(
7902                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7903                                ))
7904                            }
7905                            Err(_) => {
7906                                g.resolved = true;
7907                                g.chosen_title = None;
7908                                g.choice = Some(GithubTextChoice::Fallback);
7909                            }
7910                        }
7911                    }
7912                },
7913            }
7914            if let Some(rejected) = ask_again {
7915                // A retry (after a rejected replacement title) is still about
7916                // the original text: the same ORIGINAL / FALLBACK comparison.
7917                let w = gt::Withheld {
7918                    title: g.title,
7919                    body: g.body,
7920                    categories: g.categories.clone(),
7921                };
7922                // Named by the original text's fingerprint alone, so a resume
7923                // rewrites the same file and the detail stays identical.
7924                let artifact = match crate::run::write_artifact(
7925                    &self.state,
7926                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
7927                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7928                ) {
7929                    Ok(p) => Some(p),
7930                    Err(e) => {
7931                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
7932                        None
7933                    }
7934                };
7935                let (detail, comparison) = gt::question_content(
7936                    &w,
7937                    &pr.title,
7938                    &pr.body,
7939                    &self.state.id,
7940                    rejected.as_deref(),
7941                    artifact.as_deref(),
7942                    g.rewrite.as_ref(),
7943                );
7944                let mut q = ask::Question::new(
7945                    self.state.id.clone(),
7946                    gt::ASK_NODE.to_owned(),
7947                    gt::ASK_SEAT.to_owned(),
7948                    gt::question_summary(&self.state.config.graph.language, &w),
7949                    detail,
7950                    gt::question_choices(&w, g.rewrite.as_ref()),
7951                );
7952                q.posting_comparison = Some(comparison);
7953                q.answer_timeout = timeout;
7954                // Recorded before it is written: a stop in between leaves a
7955                // record naming a question the resume then files again.
7956                g.question = Some(q.id.clone());
7957                g.asks += 1;
7958                self.state.github_text = Some(g.clone());
7959                self.state.event(
7960                    "github-text",
7961                    format!("asking the owner about the withheld text ({})", q.short()),
7962                );
7963                self.state.save()?;
7964                store
7965                    .put_executable(&mut q)
7966                    .context("file the posting-gate question")?;
7967                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7968                    tracing::warn!(
7969                        "could not notify about posting-gate question {}: {e:#}",
7970                        q.short()
7971                    );
7972                }
7973                self.park_on_github_text(&g)?;
7974                return Ok(None);
7975            }
7976        }
7977        let resolved = g.chosen_title.clone();
7978        let original = g.choice == Some(GithubTextChoice::Original);
7979        // Field names only: the event log never records the text.
7980        self.state.event(
7981            "github-text",
7982            if original {
7983                format!(
7984                    "posting the original text for: {}",
7985                    match (g.title, g.body) {
7986                        (true, true) => "title, description",
7987                        (true, false) => "title",
7988                        _ => "description",
7989                    }
7990                )
7991            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7992                "posting the saved rewrite title and description".to_owned()
7993            } else if resolved.is_some() {
7994                "posting the owner's replacement title".to_owned()
7995            } else {
7996                "posting the neutral text for the withheld fields".to_owned()
7997            },
7998        );
7999        self.state.github_text = Some(g.clone());
8000        self.state.save()?;
8001        if g.choice == Some(GithubTextChoice::Rewrite)
8002            && let Some(r) = g.rewrite.as_ref()
8003        {
8004            return Ok(Some(PrMessage {
8005                title: r.title.clone(),
8006                body: r.body.clone(),
8007            }));
8008        }
8009        let w = gt::Withheld {
8010            title: g.title,
8011            body: g.body,
8012            categories: g.categories.clone(),
8013        };
8014        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8015        if let Some(t) = resolved {
8016            title = t;
8017        }
8018        // Every withheld field is now neutral or vetted English.
8019        let english = gt::LanguageDecision {
8020            title_english: true,
8021            body_english: true,
8022            body_complete: true,
8023        };
8024        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8025        if original {
8026            // The owner's explicit choice overrides the gate for the fields it
8027            // was made for; the others keep what the gate produced.
8028            if g.title {
8029                title = pr.title.clone();
8030            }
8031            if g.body {
8032                body = pr.body.clone();
8033            }
8034        }
8035        Ok(Some(PrMessage { title, body }))
8036    }
8037
8038    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8039        self.state.github_text = Some(g.clone());
8040        self.state.end_pass_parked(ParkCause::Question);
8041        self.state.parked = true;
8042        self.state.event(
8043            "github-text",
8044            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8045        );
8046        self.state.save()
8047    }
8048
8049    /// Ask `[roles] language_judge` about this text and record which source
8050    /// decided; the text itself is never recorded.
8051    async fn judge_pr_language(
8052        &mut self,
8053        cwd: &Path,
8054        pr: &PrMessage,
8055    ) -> Option<crate::github_text::LanguageDecision> {
8056        self.state.config.roles.language_judge.as_ref()?;
8057        if !self.state.config.graph.language_gate() {
8058            return None;
8059        }
8060        let decision =
8061            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8062        self.state.event(
8063            "github-text",
8064            if decision.is_some() {
8065                "language decided by the language judge"
8066            } else {
8067                "language judge unavailable; using built-in heuristics"
8068            },
8069        );
8070        decision
8071    }
8072
8073    async fn merge(&mut self) -> Result<()> {
8074        // Same reasoning as `gate`: ask the review and gate records directly
8075        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8076        // skip can rewrite on reentry to something that no longer says
8077        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8078        // so a hand-off (open findings, green verification) reaches merge
8079        // exactly like a genuinely clean round does.
8080        //
8081        // A run resumed mid-`land` never reaches here at all: `execute`
8082        // recognises `RunStatus::Landing` before it even calls `prep`, and
8083        // routes straight to `run_land` instead. That has to happen a level
8084        // up from this function, not with a check in here, because
8085        // `review_loop`'s own status recomputation (see its doc) runs
8086        // *before* `merge` on every reentry and would otherwise overwrite
8087        // the `Landing` marker with `Gating` before this node ever saw it.
8088        if self
8089            .state
8090            .base_sync
8091            .as_ref()
8092            .is_some_and(|s| s.conflict.is_some())
8093            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8094                != Some(RunStatus::Gating)
8095            // `gate_ran == false` is not "passed" - `gate` leaves it false
8096            // both before it has ever run and when its last attempt was
8097            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8098            // permission to merge on nothing but the review record. Only a
8099            // gate that actually ran may proceed; an intentional local
8100            // review can also finish with verification explicitly not configured.
8101            // `RunState::gate_status` computes that distinction.
8102            || !self.state.gate_allows_progress()
8103        {
8104            return Ok(());
8105        }
8106        // This node's own record, not `status`: `status == Ready` is not
8107        // unique to the harmless `MergeMode::None` path this line was
8108        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8109        // run's PR was closed without merging — and on that run `mode` is
8110        // still `Pr`, so a reentry that fell through here would push and
8111        // open a second pull request. `self.state.merge` is set exactly once
8112        // this node (or `land`) has already produced a verdict, under every
8113        // mode, which is what "already done" actually means here.
8114        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8115            return Ok(());
8116        }
8117        let Some(winner) = self.state.winner().cloned() else {
8118            return Ok(());
8119        };
8120        let repo = self.state.repo.clone();
8121        let base = self.state.base_branch.clone();
8122        let mode = self.state.config.merge.mode;
8123        let style = self.state.config.merge.style;
8124        let facts = if is_review_run(&self.state) {
8125            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8126            let start = review_base(
8127                &repo,
8128                &self.state.config.merge.remote,
8129                &base,
8130                &self.state.base_commit,
8131                &winner.branch,
8132            )
8133            .await;
8134            branch_facts(&repo, &start, &winner.branch).await
8135        } else {
8136            None
8137        };
8138        // `None` when the base could not be freshly read: then an adopted
8139        // pull request's title is left alone.
8140        let leaked = if is_review_run(&self.state) {
8141            leaked_subjects(&self.state, &winner.branch).await
8142        } else {
8143            Some(Vec::new())
8144        };
8145        // `None`: parked on the owner's word about a withheld title or body.
8146        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8147        // the resume arrives here again.
8148        let Some(pr) = self
8149            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8150            .await?
8151        else {
8152            return Ok(());
8153        };
8154        let message = pr.commit_message();
8155
8156        let outcome = match mode {
8157            MergeMode::None => MergeOutcome {
8158                mode,
8159                ok: true,
8160                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8161                empty: false,
8162            },
8163            MergeMode::Pr | MergeMode::Local
8164                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8165            {
8166                MergeOutcome {
8167                    mode,
8168                    ok: false,
8169                    detail: empty_candidate_detail(&self.state, &base),
8170                    empty: true,
8171                }
8172            }
8173            MergeMode::Local => {
8174                let on = git::current_branch(&repo).await?;
8175                if on.as_deref() != Some(base.as_str()) {
8176                    MergeOutcome {
8177                        mode,
8178                        ok: false,
8179                        detail: format!(
8180                            "{} has {} checked out, not the base branch {base}",
8181                            repo.display(),
8182                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8183                        ),
8184                        empty: false,
8185                    }
8186                } else if !git::is_clean(&repo).await? {
8187                    MergeOutcome {
8188                        mode,
8189                        ok: false,
8190                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8191                        empty: false,
8192                    }
8193                } else {
8194                    let out = match style {
8195                        MergeStyle::Merge => {
8196                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8197                        }
8198                        MergeStyle::Squash => {
8199                            git::merge_squash(&repo, &winner.branch, &message).await?
8200                        }
8201                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8202                    };
8203                    MergeOutcome {
8204                        mode,
8205                        ok: out.ok(),
8206                        detail: if out.ok() { out.stdout } else { out.stderr },
8207                        empty: false,
8208                    }
8209                }
8210            }
8211            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8212                self.state.pr_publication_pending = true;
8213                self.state.save()?;
8214                let out =
8215                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8216                self.state.pr_publication_pending = out.is_err();
8217                MergeOutcome {
8218                    mode,
8219                    ok: out.is_ok(),
8220                    detail: match out {
8221                        Ok(url) => url,
8222                        Err(e) => format!("{e:#}"),
8223                    },
8224                    empty: false,
8225                }
8226            }
8227            MergeMode::Pr => {
8228                let remote = self.state.config.merge.remote.clone();
8229                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8230                if !pushed.ok() {
8231                    MergeOutcome {
8232                        mode,
8233                        ok: false,
8234                        detail: pushed.stderr,
8235                        empty: false,
8236                    }
8237                } else {
8238                    // A retry or resume of a run whose branch already has an
8239                    // open pull request adopts it rather than failing on a
8240                    // duplicate. Only this winner branch into this base:
8241                    // `branch_for` derives the name from the run id, so a
8242                    // different run's pull request never matches.
8243                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8244                    let out = match pr_merge_plan(found) {
8245                        PrPlan::Create => {
8246                            gh_pr_create(
8247                                &winner.worktree,
8248                                &base,
8249                                &winner.branch,
8250                                &pr.title,
8251                                &pr.body,
8252                            )
8253                            .await
8254                        }
8255                        PrPlan::Adopt { url, title } => {
8256                            self.state
8257                                .event("merge", format!("Pr: adopted open pull request {url}"));
8258                            if self.state.github_text.as_ref().is_some_and(|g| {
8259                                g.resolved
8260                                    && g.choice == Some(GithubTextChoice::Rewrite)
8261                                    && g.rewrite.is_some()
8262                            }) {
8263                                // The owner chose the entire pair, even if the
8264                                // existing title already matches or was edited.
8265                                land::set_pr_text_verbatim(
8266                                    &winner.worktree,
8267                                    &url,
8268                                    &pr.title,
8269                                    &pr.body,
8270                                )
8271                                .await
8272                                .map(|()| url)
8273                            } else {
8274                                if title != pr.title
8275                                    && (!is_review_run(&self.state)
8276                                        || leaked
8277                                            .as_deref()
8278                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8279                                    && let Err(e) =
8280                                        if self.state.github_text.as_ref().is_some_and(|g| {
8281                                            g.resolved
8282                                                && g.title
8283                                                && g.choice == Some(GithubTextChoice::Original)
8284                                        }) {
8285                                            // The owner chose to post this title unmodified.
8286                                            land::set_pr_title_verbatim(
8287                                                &winner.worktree,
8288                                                &url,
8289                                                &pr.title,
8290                                            )
8291                                            .await
8292                                        } else {
8293                                            land::set_pr_title(
8294                                                &mut self.state,
8295                                                &winner.worktree,
8296                                                &url,
8297                                                &pr.title,
8298                                            )
8299                                            .await
8300                                        }
8301                                {
8302                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8303                                    self.state
8304                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8305                                }
8306                                Ok(url)
8307                            }
8308                        }
8309                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8310                    };
8311                    match out {
8312                        Ok(url) => MergeOutcome {
8313                            mode,
8314                            ok: true,
8315                            detail: url,
8316                            empty: false,
8317                        },
8318                        Err(e) => MergeOutcome {
8319                            mode,
8320                            ok: false,
8321                            detail: e.to_string(),
8322                            empty: false,
8323                        },
8324                    }
8325                }
8326            }
8327        };
8328
8329        self.state.status = match (mode, outcome.ok) {
8330            (MergeMode::None, _) => RunStatus::Ready,
8331            (_, true) if outcome.empty => RunStatus::Ready,
8332            (MergeMode::Pr, true) => RunStatus::Ready,
8333            (_, true) => RunStatus::Merged,
8334            (_, false) => RunStatus::Blocked,
8335        };
8336        self.state.event(
8337            "merge",
8338            format!(
8339                "{:?}: {}",
8340                mode,
8341                outcome.detail.lines().next().unwrap_or("")
8342            ),
8343        );
8344        self.state.merge = Some(outcome);
8345        self.state.save()?;
8346
8347        // The PR is open and the run would historically stop here, leaving the
8348        // operator to watch checks, feed review comments back to a fixer, and
8349        // merge. That was done by hand six times in one session before this
8350        // existed. Opt-in, because merging is the one irreversible thing magi
8351        // can do to a repository.
8352        if self.state.config.graph.land
8353            && mode == MergeMode::Pr
8354            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8355        {
8356            self.run_land().await?;
8357        }
8358        // `run_land` may have left `status` at `Landing` - still waiting on
8359        // CI or the owner's approval, not actually settled - so this has to
8360        // read whatever `status` ended up as here, not the `Merged` this
8361        // function set a few lines up.
8362        self.settle_questions();
8363        Ok(())
8364    }
8365
8366    /// Enter `land`.
8367    ///
8368    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8369    /// a resumed run's re-entry. `land::land` itself is what serialises the
8370    /// two git-mutating moments inside the loop — the rebase push and
8371    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8372    /// to hold a lock across the whole call, and doing so would serialise
8373    /// this run's CI wait against a *different* run's land-approval resume
8374    /// in the same repository, which is exactly the "must not wait on
8375    /// another task" property the daemon's slot-freeing exists to give.
8376    async fn run_land(&mut self) -> Result<()> {
8377        crate::pr_head::ensure_owner(&self.state)?;
8378        let url = self
8379            .state
8380            .pr
8381            .as_ref()
8382            .map(|pr| pr.url.clone())
8383            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8384            .unwrap_or_default();
8385        let url = url.lines().next().unwrap_or("").trim().to_owned();
8386        if !url.starts_with("http") {
8387            return Ok(());
8388        }
8389        // A land failure is not a lost run: the work is on a branch and the
8390        // pull request is open, which is exactly where a human takes over.
8391        loop {
8392            match land::land(&mut self.state, &url).await {
8393                Ok(_)
8394                    if self
8395                        .state
8396                        .merge_repairs
8397                        .last()
8398                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8399                {
8400                    if self.repair_merge_review().await? {
8401                        continue;
8402                    }
8403                }
8404                Ok(pr) if self.state.parked => {
8405                    // `land` already saved the parked marker; nothing here
8406                    // overrides `status` back to a terminal value while an
8407                    // approval is still outstanding.
8408                    let _ = pr;
8409                }
8410                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8411                    self.state.status = RunStatus::Superseded;
8412                    self.state.save()?;
8413                }
8414                Ok(pr) => {
8415                    self.state.status = match pr.state {
8416                        land::PrLifecycle::Merged => RunStatus::Merged,
8417                        _ => RunStatus::Blocked,
8418                    };
8419                    // Downstream of a confirmed merge only - see
8420                    // `bump::should_release_bump`'s own doc for why this one
8421                    // check covers all three of `land`'s success paths.
8422                    // Best-effort: the run already landed, so a failure here
8423                    // (the decision call, `gh`, `cargo`) is recorded and never
8424                    // turns a landed run into a failed one.
8425                    if bump::should_release_bump(self.state.status)
8426                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8427                    {
8428                        // The event is the run's own record. Not-eligible cases
8429                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8430                        // `Err` is a bump that was tried and failed:
8431                        // `after_merge` itself raises the operator notice for
8432                        // that, whether or not a release PR exists yet.
8433                        self.state
8434                            .event("bump", format!("release bump skipped: {e:#}"));
8435                    }
8436                    // Independent of the bump, and best-effort in the same way:
8437                    // findings the merge left open become follow-up tasks.
8438                    if self.state.status == RunStatus::Merged {
8439                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8440                    }
8441                    self.state.save()?;
8442                }
8443                Err(e) => {
8444                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8445                        return Err(e);
8446                    }
8447                    self.state.status = RunStatus::Blocked;
8448                    self.state.event("land", format!("gave up: {e}"));
8449                    self.state.save()?;
8450                }
8451            }
8452            break;
8453        }
8454        Ok(())
8455    }
8456
8457    /// One repair cycle stays under the existing driver/task claim. An
8458    /// interrupted fixer is rolled back and held rather than run twice.
8459    async fn repair_merge_review(&mut self) -> Result<bool> {
8460        use crate::run::MergeRepairPhase as Phase;
8461        crate::pr_head::ensure_owner(&self.state)?;
8462        let i = self.state.merge_repairs.len() - 1;
8463        let request = self.state.merge_repairs[i].clone();
8464        if request.phase == Phase::Fixing {
8465            let restored = match self.state.winner().cloned() {
8466                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8467                None => Err(anyhow::anyhow!("repair has no winning branch")),
8468            };
8469            let why = match restored {
8470                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8471                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8472                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8473            };
8474            self.hold_merge_repair(i, why)?;
8475            return Ok(false);
8476        }
8477        let failure = if i >= self.state.config.graph.land_rounds
8478            || self.state.config.graph.review_rounds == 0
8479        {
8480            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8481        } else {
8482            None
8483        };
8484        if let Some(why) = failure {
8485            self.hold_merge_repair(i, why)?;
8486            return Ok(false);
8487        }
8488        if request.phase == Phase::Requested {
8489            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8490                Ok(pr) => pr,
8491                Err(e) => {
8492                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8493                    return Ok(false);
8494                }
8495            };
8496            self.state.merge_repairs[i].phase = Phase::Fixing;
8497            self.state.land_approval = None;
8498            self.state.land_armed_head = None;
8499            self.state.save()?;
8500            // Serialize the exact records, including finding ids/details,
8501            // reconsideration reasons, final votes and verification output.
8502            let evidence = serde_json::to_string_pretty(&request)?;
8503            let reason = format!(
8504                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8505            );
8506            let budget = self.state.config.graph.land_rounds;
8507            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8508            match result {
8509                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8510                Ok(land::Fixed::Failed(why)) => {
8511                    self.hold_merge_repair(
8512                        i,
8513                        format!(
8514                            "repair failed: {why}; inspect the branch and run an explicit review"
8515                        ),
8516                    )?;
8517                    return Ok(false);
8518                }
8519                Err(e) => {
8520                    self.hold_merge_repair(
8521                        i,
8522                        format!(
8523                            "repair failed: {e:#}; inspect the branch and run an explicit review"
8524                        ),
8525                    )?;
8526                    return Ok(false);
8527                }
8528            }
8529            // Even no change gets a real new reviewer wave. The previous
8530            // evidence remains in the request; a former clean/concluded round
8531            // cannot short-circuit the new review or gate.
8532            self.begin_merge_repair_review(i)?;
8533        }
8534        self.review_loop().await?;
8535        self.gate().await?;
8536        if self.resource_halt()? || self.state.parked {
8537            return Ok(false);
8538        }
8539        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8540            != Some(RunStatus::Gating)
8541            || !self.state.gate_allows_progress()
8542        {
8543            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8544            return Ok(false);
8545        }
8546        // Review and gate may have added commits after the initial repair.
8547        // Publish exactly their final head to the existing PR, with a lease.
8548        let winner = self
8549            .state
8550            .winner()
8551            .cloned()
8552            .context("repair has no winning branch")?;
8553        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8554        if self.state.gate_head.as_deref() != Some(after.as_str())
8555            || !git::is_clean(&winner.worktree).await?
8556        {
8557            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8558            return Ok(false);
8559        }
8560        let expected = self
8561            .state
8562            .pr_pushed_head
8563            .as_deref()
8564            .unwrap_or(&request.head)
8565            .to_owned();
8566        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8567        // Push the checked commit itself: a local ref moving after the check
8568        // must not change which commit the forge receives.
8569        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8570        let refspec = format!("{after}:refs/heads/{public}");
8571        let push = git::git_raw(
8572            &winner.worktree,
8573            &["push", &lease, &self.state.config.merge.remote, &refspec],
8574        )
8575        .await;
8576        match push {
8577            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8578            Ok(out) => {
8579                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8580                return Ok(false);
8581            }
8582            Err(e) => {
8583                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8584                return Ok(false);
8585            }
8586        }
8587        self.state.merge_repairs[i].phase = Phase::Complete;
8588        self.state.status = RunStatus::Landing;
8589        self.state.event(
8590            "repair",
8591            "new review and gate completed; fresh owner merge decision required",
8592        );
8593        self.state.save()?;
8594        Ok(true)
8595    }
8596
8597    fn merge_repair_artifacts(&self) -> PathBuf {
8598        let artifacts = agent::artifacts_dir(&self.state.dir());
8599        match self
8600            .state
8601            .merge_repairs
8602            .last()
8603            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8604        {
8605            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8606            None => artifacts,
8607        }
8608    }
8609
8610    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8611        self.state.reviews.clear();
8612        self.state.gate.clear();
8613        self.state.gate_ran = false;
8614        self.state.gate_head = None;
8615        self.state.gate_fixes.clear();
8616        self.state.gate_repair_e2e = None;
8617        self.state.pre_gate.clear();
8618        self.state.pre_gate_commit = None;
8619        self.state.contested_handoff = None;
8620        self.state.land_reruns = None;
8621        self.state.land_approval = None;
8622        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8623        self.state.status = RunStatus::Reviewing;
8624        self.state.save()
8625    }
8626
8627    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8628        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8629        self.state.merge_repairs[index].failure = Some(why.clone());
8630        self.state.status = RunStatus::Blocked;
8631        self.state.event("repair", why);
8632        self.state.save()
8633    }
8634
8635    // -------------------------------------------------------------- helpers
8636
8637    /// Fetch or create a seat, keeping its conversation across nodes.
8638    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8639        if let Some(existing) = self.state.seats.get(key)
8640            && existing.agent == agent
8641        {
8642            return existing.clone();
8643        }
8644        // A seat that changes agent mints its session id from the agent too,
8645        // like a handover: the old agent's uuid is already taken by the CLI.
8646        let fresh = if self.state.seats.contains_key(key) {
8647            handover_seat(key, agent, self.state.next_seat_seed())
8648        } else {
8649            SeatState::new(key, agent, self.state.seed)
8650        };
8651        self.state.seats.insert(key.to_owned(), fresh.clone());
8652        fresh
8653    }
8654
8655    /// The agent now holding seat `key`: `spec`, unless a handover moved the
8656    /// seat to another roster agent, in which case that agent. Nodes that
8657    /// continue a seat's conversation (deliberation, the votes, a reviewer's
8658    /// reconsideration) must keep talking to whoever answered it, not slip
8659    /// back to the agent that failed it.
8660    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8661        match self.state.seats.get(key) {
8662            Some(s) if s.agent != spec.id => {
8663                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8664            }
8665            _ => spec,
8666        }
8667    }
8668
8669    /// A candidate rendered for judging, with the leak policy applied.
8670    fn view(&self, c: &Candidate) -> CandidateView {
8671        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8672            .unwrap_or_default();
8673        let (patch, _) = blind::sanitize_patch(
8674            &format!("candidate {} patch", c.label),
8675            &raw,
8676            &self.state.config.blind,
8677        );
8678        CandidateView {
8679            label: c.label,
8680            branch: c.branch.clone(),
8681            summary: c.summary.clone(),
8682            stat: c.stat.clone(),
8683            patch,
8684        }
8685    }
8686
8687    /// The full candidate set as prompt text, for seats with no live session.
8688    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8689        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8690        prompt::judge(
8691            "(see above)",
8692            &views,
8693            self.roles.judges.len(),
8694            base_short,
8695            "en",
8696        )
8697    }
8698
8699    /// The final-vote prompt with everything a seat that has no session of its
8700    /// own needs: the candidates, the seat's own ranking and reasons, and the
8701    /// anonymised deliberation it took part in (only when there was one, so a
8702    /// handed-over seat never sees more than the seat it replaces did). The
8703    /// `Final vote` heading stays first.
8704    fn vote_prompt_full(
8705        &self,
8706        j: usize,
8707        labels: &[char],
8708        language: &str,
8709        candidates: &[Candidate],
8710        base_short: &str,
8711    ) -> String {
8712        let mut text = format!(
8713            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8714            prompt::final_vote(labels, language),
8715            self.state.instruction,
8716            self.candidate_block(candidates, base_short)
8717        );
8718        if let Some(own) = self
8719            .state
8720            .judgements
8721            .get(j)
8722            .filter(|r| !r.ranking.is_empty())
8723        {
8724            let reasons = own
8725                .reasons
8726                .iter()
8727                .map(|(k, v)| format!("- {k}: {v}"))
8728                .collect::<Vec<_>>()
8729                .join("\n");
8730            text.push_str(&format!(
8731                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8732                own.ranking.iter().collect::<String>(),
8733                if reasons.is_empty() {
8734                    ""
8735                } else {
8736                    ", because:\n"
8737                }
8738            ));
8739        }
8740        if !self.state.deliberation.is_empty() {
8741            text.push_str("\n# What was argued before this vote\n");
8742            for t in self.transcript(&[], j) {
8743                text.push_str(&format!(
8744                    "\n## {}{}\n\n{}\n",
8745                    t.who,
8746                    if t.is_self { " (you)" } else { "" },
8747                    t.body.trim()
8748                ));
8749            }
8750        }
8751        text
8752    }
8753
8754    /// Anonymised transcript for judge `self_idx`.
8755    ///
8756    /// The initial rankings are always the opening statements. Seeding them
8757    /// only when no turn had been taken yet meant every judge after the first
8758    /// argued against a single voice instead of against the actual split — the
8759    /// disagreement is the information, so it is always on the table.
8760    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8761        let mut turns = Vec::new();
8762        for j in &self.state.judgements {
8763            if j.ranking.is_empty() {
8764                continue;
8765            }
8766            let reasons = j
8767                .reasons
8768                .iter()
8769                .map(|(k, v)| format!("- {k}: {v}"))
8770                .collect::<Vec<_>>()
8771                .join("\n");
8772            turns.push(Turn {
8773                who: format!("Judge {} (opening ranking)", j.judge),
8774                is_self: j.judge == self_idx + 1,
8775                body: format!(
8776                    "Ranked {}{}{reasons}",
8777                    j.ranking.iter().collect::<String>(),
8778                    if reasons.is_empty() {
8779                        ""
8780                    } else {
8781                        ", because:\n"
8782                    }
8783                ),
8784            });
8785        }
8786        for t in self
8787            .state
8788            .deliberation
8789            .iter()
8790            .flat_map(|r| r.turns.iter())
8791            .chain(current)
8792        {
8793            turns.push(Turn {
8794                who: format!("Judge {}", t.judge),
8795                is_self: t.judge == self_idx + 1,
8796                body: t.body.clone(),
8797            });
8798        }
8799        turns
8800    }
8801}
8802
8803/// Does this seat still hold the context a follow-up prompt would rely on?
8804fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8805    agent::has_session(spec.kind, seat, sessions)
8806}
8807
8808/// The next entry in `roster` after `start`, never wrapping back to the
8809/// front, whose id is not in `tried` yet.
8810///
8811/// Starts one past `start` rather than at the front of `roster`: `start` is
8812/// the seat's own original position, and a seat whose candidate slot already
8813/// sits on the roster's second entry must fall through to the third next, not
8814/// restart at the first — which is very likely a different candidate's own
8815/// agent already. Never wraps back past `start`, for the same reason: an
8816/// entry earlier in the roster than the seat's own position is almost
8817/// certainly some *other* candidate slot's own agent, and once the tail of
8818/// the roster is exhausted there are no more untried agents for *this* seat
8819/// to fall through to — the caller's fallback chain ends there, exactly as
8820/// "no further untried agents remain in the list for that seat" asks for.
8821///
8822/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
8823/// the same id twice (an operator's `roles.implementers` typo, or a
8824/// `[[agents]]` list reused across roles) must not let
8825/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
8826/// over `roster` either finds an untried id or runs out, so this always
8827/// terminates regardless of duplicates.
8828fn next_untried_in_roster<'a>(
8829    roster: &'a [AgentSpec],
8830    start: usize,
8831    tried: &BTreeSet<String>,
8832) -> Option<&'a AgentSpec> {
8833    roster
8834        .get(start + 1..)?
8835        .iter()
8836        .find(|s| !tried.contains(&s.id))
8837}
8838
8839/// The successor for a seat, shared by all four seat kinds (implement, judge,
8840/// review, advise). `others` holds the ids that *currently* occupy the other
8841/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
8842/// sees them), so roster entries beyond the seat count act as spares: a failed
8843/// seat goes to an agent no other seat holds whenever the roster permits.
8844///
8845/// `carried` is `Some` only for the review loop's carried failure history.
8846/// Order: (1) forward from `start`, never wrapping, untried, not a carried
8847/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
8848/// whole roster: untried and not another seat's occupant; (3) the plain walk
8849/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
8850/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
8851/// a better outcome than an empty seat, and it keeps the answer to "is there
8852/// a successor at all" exactly what it was before occupants were considered,
8853/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
8854/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
8855/// carried failure is a cheaper bet than doubling an agent on the panel, and
8856/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
8857/// the same round are handled one at a time, so a seat later in the batch
8858/// sees an earlier one's new occupant but not yet its own freed agent.
8859fn pick_successor<'a>(
8860    roster: &'a [AgentSpec],
8861    start: usize,
8862    tried: &BTreeSet<String>,
8863    carried: Option<&BTreeSet<String>>,
8864    others: &BTreeSet<String>,
8865) -> Option<&'a AgentSpec> {
8866    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8867    roster
8868        .get(start + 1..)
8869        .and_then(|tail| {
8870            tail.iter()
8871                .filter(free)
8872                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8873        })
8874        .or_else(|| {
8875            carried
8876                .is_some()
8877                .then(|| roster.iter().find(free))
8878                .flatten()
8879        })
8880        .or_else(|| match carried {
8881            Some(c) => next_for_seat(roster, start, tried, c),
8882            None => next_untried_in_roster(roster, start, tried),
8883        })
8884}
8885
8886/// The next agent for a seat that carries its failure history across rounds
8887/// (the review loop). `round_tried` is this round's own bound and starts
8888/// empty every round; `carried_failed` only decides priority.
8889///
8890/// First: an id walking forward from `start`, never wrapping, that is neither
8891/// tried this round nor failed in an earlier one. Only when that is exhausted
8892/// does it rescue: the first roster id (in roster order, so this one *does*
8893/// look before `start`) not yet tried this round, which is by then a carried
8894/// failure. Each id is rescued at most once per round, so it cannot loop.
8895fn next_for_seat<'a>(
8896    roster: &'a [AgentSpec],
8897    start: usize,
8898    round_tried: &BTreeSet<String>,
8899    carried_failed: &BTreeSet<String>,
8900) -> Option<&'a AgentSpec> {
8901    roster
8902        .get(start + 1..)?
8903        .iter()
8904        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8905        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8906}
8907
8908/// Where a reviewer seat starts a round: the agent that last answered it when
8909/// it is still on the roster and not marked failed, else the spec's own agent
8910/// unless it failed, else the next roster agent that has not failed, else the
8911/// spec's own agent again (the whole roster failed). Ids no longer on the
8912/// roster are ignored. An empty roster has no handover, so the spec stands.
8913fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8914    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8915        return spec;
8916    };
8917    let ok = |id: &str| !h.failed.contains(id);
8918    if let Some(last) = h.last_ok.as_deref()
8919        && ok(last)
8920        && let Some(s) = roster.iter().find(|s| s.id == last)
8921    {
8922        return s.clone();
8923    }
8924    if ok(&spec.id) {
8925        return spec;
8926    }
8927    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8928    roster
8929        .get(from + 1..)
8930        .into_iter()
8931        .flatten()
8932        .chain(roster.iter())
8933        .find(|s| ok(&s.id))
8934        .cloned()
8935        .unwrap_or(spec)
8936}
8937
8938/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
8939/// seed so a CLI that mints its session id up front (`--session-id`) never
8940/// reuses the uuid the previous agent already opened under the same seat key.
8941pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8942    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8943}
8944
8945/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
8946/// so the classifier below and the code that builds the message cannot drift.
8947const TIMED_OUT: &str = "timed out";
8948
8949impl FailClass {
8950    /// `None` for an answer; otherwise how the turn failed.
8951    fn of(out: &AgentOutcome) -> Option<Self> {
8952        match out {
8953            AgentOutcome::Ok(_) => None,
8954            AgentOutcome::Quota(_) => Some(Self::Quota),
8955            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8956            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8957            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8958            // Not a failure of the agent. Callers check for it first; `None`
8959            // here means "no class to hand over on".
8960            AgentOutcome::ResourceBlocked(_) => None,
8961        }
8962    }
8963
8964    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
8965    fn stem_word(&self) -> &'static str {
8966        match self {
8967            Self::Quota => "quota",
8968            _ => "handover",
8969        }
8970    }
8971}
8972
8973/// The message's first line with its variable parts removed — digit runs and
8974/// path-like tokens — so "exit code 2" and "exit code 7" read
8975/// as one kind of failure.
8976fn failure_signature(msg: &str) -> String {
8977    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8978    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
8979    // the historical machine key while the process diagnosis uses readable prose.
8980    if let Some(code) = line
8981        .strip_prefix("process exited with no usable output (exit code ")
8982        .and_then(|s| s.strip_suffix(')'))
8983        .and_then(|s| s.parse::<i32>().ok())
8984    {
8985        line = format!("exited with some({code}) and no usable output");
8986    } else if matches!(
8987        line.as_str(),
8988        "process exited with no usable output (no exit code reported)"
8989            | "process exited with no usable output (exit code unavailable)"
8990    ) {
8991        line = "exited with none and no usable output".to_owned();
8992    }
8993    let mut out = Vec::new();
8994    for word in line.split_whitespace() {
8995        if word.contains('/') || word.contains('\\') {
8996            out.push("<path>".to_owned());
8997            continue;
8998        }
8999        let mut w = String::new();
9000        let mut in_digits = false;
9001        for c in word.chars() {
9002            if c.is_ascii_digit() {
9003                if !in_digits {
9004                    w.push('#');
9005                }
9006                in_digits = true;
9007            } else {
9008                in_digits = false;
9009                w.push(c);
9010            }
9011        }
9012        out.push(w);
9013    }
9014    out.join(" ").chars().take(120).collect()
9015}
9016
9017/// Whether a seat that just failed with `cur` may go to the next roster agent.
9018/// A quota or a timeout always may. Any other failure may not when the agent
9019/// before it failed the same way: an error the prompt causes would otherwise
9020/// walk the whole roster. `prev` is the class of the immediately preceding
9021/// agent's failure, so a quota or timeout in between breaks the run of
9022/// identical failures by itself.
9023fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9024    match cur {
9025        FailClass::Quota | FailClass::Timeout => true,
9026        FailClass::Other(_) => prev != Some(cur),
9027    }
9028}
9029
9030/// A short human reason for a failed outcome, for the handover record.
9031fn fail_reason(out: &AgentOutcome) -> String {
9032    match out {
9033        AgentOutcome::Ok(_) => String::new(),
9034        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9035        AgentOutcome::Dropped(o) => format!(
9036            "the CLI dropped the stream ({})",
9037            o.dropped
9038                .as_ref()
9039                .map(|d| d.why.as_str())
9040                .unwrap_or("it ended without delivering its answer")
9041        ),
9042        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9043            e.lines().next().unwrap_or("").chars().take(160).collect()
9044        }
9045    }
9046}
9047
9048/// Note one handover in the run: the structured record and, in the timeline,
9049/// the sentence a person reads. A quota keeps the wording it always had.
9050pub(crate) fn record_handover(
9051    state: &mut RunState,
9052    node: &str,
9053    seat: &str,
9054    from: &str,
9055    to: &str,
9056    class: &FailClass,
9057    reason: &str,
9058) {
9059    let message = if *class == FailClass::Quota {
9060        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9061    } else {
9062        format!("{seat}: handed over {from} -> {to} ({reason})")
9063    };
9064    state.event(node, message);
9065    state.handovers.push(Handover {
9066        at: Timestamp::now(),
9067        node: node.to_owned(),
9068        seat: seat.to_owned(),
9069        from: from.to_owned(),
9070        to: to.to_owned(),
9071        reason: reason.to_owned(),
9072    });
9073}
9074
9075/// Did this reply report running a command whose own CLI never confirmed an
9076/// exit status?
9077///
9078/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9079/// command *finished* (see that type's own doc), so this can only be `true`
9080/// for a command whose completion event carried no readable exit code — not
9081/// for one that simply is not mentioned at all. That is the one signal this
9082/// crate can read, from the same record `state.jobs` renders, about a reply
9083/// standing next to work its own CLI cannot vouch for finishing; it is
9084/// deliberately not a check on the exit code's *value* (a fixer legitimately
9085/// runs a command that fails mid-iteration before it succeeds) and not a
9086/// guess at a command still running in the background (which emits no event
9087/// at all, and so leaves no evidence here to find).
9088fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9089    commands.iter().any(|c| c.exit_code.is_none())
9090}
9091
9092/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9093/// trusted as a verified no-op — the adoption guard's own text-level half.
9094///
9095/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9096/// timed out): a marker only earns the benefit of the doubt from a turn the
9097/// CLI itself vouches for finishing properly, the same house style
9098/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9099/// *fix* report to for `commands`. A candidate that timed out, exited
9100/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9101/// is, whatever prose it wrote — this returns `None` before it ever looks at
9102/// `text`. The remaining guards (the tree really is empty, the evidence is
9103/// non-empty) are the caller's: this only reads what the reply *claimed*.
9104fn verified_noop_claim(
9105    usable: bool,
9106    commands: &[agent::CommandEvidence],
9107    text: &str,
9108) -> Option<String> {
9109    (usable && !has_unconfirmed_command(commands))
9110        .then(|| verdict::verified_noop(text))
9111        .flatten()
9112}
9113
9114fn short(commit: &str) -> String {
9115    commit.chars().take(7).collect()
9116}
9117
9118fn make_executable(path: &Path) -> Result<()> {
9119    #[cfg(unix)]
9120    {
9121        use std::os::unix::fs::PermissionsExt as _;
9122        let mut perms = std::fs::metadata(path)?.permissions();
9123        perms.set_mode(0o755);
9124        std::fs::set_permissions(path, perms)?;
9125    }
9126    #[cfg(not(unix))]
9127    {
9128        let _ = path;
9129    }
9130    Ok(())
9131}
9132
9133/// What every seat in one batch shares: where the answers are attributed, the
9134/// prompt overlay they inherit, and the build cache they are told to use.
9135///
9136/// A struct rather than four more parameters: `wave` also needs the run's
9137/// state (to record who is answering right now) and the attempt number, and
9138/// eight positional arguments is both unreadable and a clippy error.
9139struct WaveCtx<'a> {
9140    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9141    /// paid for it.
9142    run: &'a str,
9143    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9144    node: &'a str,
9145    prompts: &'a Prompts,
9146    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9147    cache: Option<&'a Path>,
9148    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9149    /// `JobRecord::round`. `None` for every other node.
9150    round: Option<usize>,
9151    /// Carry each seat's failed-agent history across waves (the review loop
9152    /// only): start-of-round priority and handover choice read
9153    /// [`RunState::seat_history`], and every answer or failure writes it.
9154    carry_seats: bool,
9155    /// The run's park / interrupt requests, so a wait for a shared resource
9156    /// can be abandoned the moment the operator asks to park.
9157    pauses: Option<(&'a Pause, &'a Pause)>,
9158}
9159
9160/// Shared scheduling policy for operations and spawned seat invocations.
9161fn repository_wait(
9162    state: &RunState,
9163    pauses: Option<(&Pause, &Pause)>,
9164) -> crate::repo_lock::WaitPolicy {
9165    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9166    let inherited = crate::repo_lock::WaitPolicy::current();
9167    crate::repo_lock::WaitPolicy::new(
9168        Duration::from_secs(state.config.graph.timeout_resource_wait),
9169        move || {
9170            inherited.cancelled()
9171                || pauses
9172                    .as_ref()
9173                    .is_some_and(|(a, b)| a.parked() || b.parked())
9174        },
9175    )
9176}
9177
9178fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9179    if error.is::<crate::repo_lock::ResourceBlocked>() {
9180        state.resource_block = Some(crate::run::ResourceBlock {
9181            node: "repository".into(),
9182            kind: if error.to_string().contains("disk pressure") {
9183                "disk pressure"
9184            } else {
9185                "repository coordination"
9186            }
9187            .into(),
9188            detail: error.to_string(),
9189            at: Timestamp::now(),
9190        });
9191        state.event("resource", error.to_string());
9192    }
9193}
9194
9195async fn coordinate_operation(
9196    state: &mut RunState,
9197    cwd: &Path,
9198    wait: &crate::repo_lock::WaitPolicy,
9199) -> Result<Option<crate::repo_lock::SeatLease>> {
9200    let result =
9201        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9202            .await;
9203    if let Err(e) = &result {
9204        record_repository_block(state, e);
9205    }
9206    result
9207}
9208
9209/// Run one job, honouring the parallelism budget.
9210async fn run_one(
9211    job: SeatJob,
9212    sem: Arc<Semaphore>,
9213    ctx: &WaveCtx<'_>,
9214    state: &mut RunState,
9215    attempt: usize,
9216) -> (SeatState, AgentOutcome) {
9217    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9218        .await
9219        .pop()
9220        .expect("one job in, one result out");
9221    (seat, out)
9222}
9223
9224/// Run every job concurrently, capped by the semaphore, preserving order.
9225///
9226/// Every seat in the batch is recorded into [`RunState::active`] before the
9227/// wave starts and cleared as each answer lands, so the run's own record says
9228/// who is still being waited on rather than only who finished.
9229async fn wave(
9230    jobs: Vec<SeatJob>,
9231    sem: Arc<Semaphore>,
9232    ctx: &WaveCtx<'_>,
9233    state: &mut RunState,
9234    attempt: usize,
9235) -> Vec<(usize, SeatState, AgentOutcome)> {
9236    let WaveCtx {
9237        run,
9238        node,
9239        prompts,
9240        cache,
9241        round,
9242        carry_seats: _,
9243        pauses,
9244    } = *ctx;
9245    for job in &jobs {
9246        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9247    }
9248    if let Err(e) = state.save() {
9249        // A failed persist of "who is answering right now" must not abort the
9250        // wave: the seats are already being asked, and the alternative is
9251        // losing the answers to save a status line nobody may even be
9252        // watching.
9253        tracing::warn!("could not persist in-progress seats: {e:#}");
9254    }
9255    // Hold the shared build cache's lease for the whole batch, not per job:
9256    // several candidates (an implement wave) or a fixer legitimately share
9257    // one cache concurrently within this run, and that stays untouched — a
9258    // single lease taken once for the whole wave and released once it is
9259    // done is what stops a *different* borrower (another run's own wave, its
9260    // e2e/gate, a human's `magi review`) from interleaving a build into the
9261    // same directory while this one is in flight. Best-effort, not
9262    // all-or-nothing: a wave that cannot get the lease within its own
9263    // longest job's budget still runs — an hour of paid implementer calls is
9264    // not thrown away over cache contention — but every write-allowed seat
9265    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9266    // below), the same fallback a read-only seat always gets, rather than
9267    // building into a directory this run was never granted. The identity
9268    // record is still invalidated below either way, so the next tracked
9269    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9270    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9271    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9272    // never borrowed from a job's agent budget (and never charged to it either,
9273    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9274    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9275    let cache_guard = if let Some(cache_dir) = cache {
9276        if jobs_had_a_writer {
9277            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9278            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9279            match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9280            {
9281                Ok(Some(g)) => Some(g),
9282                // Parking: leave without spawning anything. The run parks at
9283                // the next boundary and the seats are asked afresh on resume.
9284                Ok(None) => {
9285                    state.resource_block = Some(crate::run::ResourceBlock {
9286                        node: node.to_owned(),
9287                        kind: "shared build cache".to_owned(),
9288                        detail: "the run is parking".to_owned(),
9289                        at: Timestamp::now(),
9290                    });
9291                    return block_wave(
9292                        jobs,
9293                        state,
9294                        node,
9295                        attempt,
9296                        "parked while waiting for the shared build cache",
9297                    );
9298                }
9299                // An unreadable lease is not contention: keep the old
9300                // fallback (no shared cache for this wave's seats).
9301                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9302                Err(e) => {
9303                    // Writing seats may not build into a cache this run does
9304                    // not hold, and an agent that cannot be given one is not
9305                    // asked at all: the run stops resumably instead.
9306                    let why = format!("{e:#}");
9307                    state.resource_block = Some(crate::run::ResourceBlock {
9308                        node: node.to_owned(),
9309                        kind: "shared build cache".to_owned(),
9310                        detail: why.clone(),
9311                        at: Timestamp::now(),
9312                    });
9313                    state.event(
9314                        "resource",
9315                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9316                    );
9317                    return block_wave(jobs, state, node, attempt, &why);
9318                }
9319            }
9320        } else {
9321            None
9322        }
9323    } else {
9324        None
9325    };
9326    // Every job keeps its whole budget: the wait above was not the agent's.
9327    // Restart the seats' clocks so "started" means "being spawned now".
9328    state.seats_spawning(node, attempt);
9329    if let Err(e) = state.save() {
9330        tracing::warn!("could not persist the start of a wave: {e:#}");
9331    }
9332    // Queueing behind this wave's own seats for a parallelism slot is normal
9333    // (each of them is bounded by its own agent timeout); only a wait beyond
9334    // all of them plus the resource limit means the slots never came back.
9335    let slot_limit = resource_limit
9336        + jobs
9337            .iter()
9338            .map(|j| j.timeout)
9339            .max()
9340            .unwrap_or_default()
9341            .saturating_mul(jobs.len() as u32);
9342    let mut set = tokio::task::JoinSet::new();
9343    // Seats announce themselves here the moment they hold a parallelism
9344    // permit, i.e. just before their CLI is spawned.
9345    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9346    let overlay = prompts.overlay(node);
9347    let repository_wait = repository_wait(state, pauses);
9348    for (i, mut job) in jobs.into_iter().enumerate() {
9349        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9350        if cache.is_some() {
9351            job.prompt.push('\n');
9352            job.prompt
9353                .push_str(&prompt::build_cache_note(node, job.allow_write));
9354        }
9355        let sem = Arc::clone(&sem);
9356        let run = run.to_owned();
9357        let node = node.to_owned();
9358        // Only implementers were told about the task's attachments, so only
9359        // their seats get the directory widened for reading.
9360        let attachments = if node == "implement" {
9361            state.attachments.clone()
9362        } else {
9363            Vec::new()
9364        };
9365        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9366        // `prompt::build_cache_note`'s doc for why setting it anyway is
9367        // exactly how a sandboxed reviewer's write refusal got reported as a
9368        // defect in the patch, not a property of its own seat. And a
9369        // write-allowed one is handed it only when the lease above was
9370        // actually acquired: a wave that could not get it (`cache_guard` is
9371        // `None`, see its own comment) must not send seats to build into a
9372        // directory this run does not hold - that is the exact concurrent,
9373        // unmanaged-write race this module exists to prevent, not something
9374        // "proceeding anyway" is allowed to reintroduce.
9375        let cache = cache
9376            .filter(|_| job.allow_write && cache_guard.is_some())
9377            .map(Path::to_path_buf);
9378        let spawned_tx = spawned_tx.clone();
9379        let repository_wait = repository_wait.clone();
9380        let coordination = state.config.coordination.clone();
9381        let fetched_base = state.fetched_base.clone();
9382        set.spawn(async move {
9383            // A wait for a parallelism slot is scheduling, not the agent: it
9384            // is bounded by its own limit and ends as a resource block, never
9385            // as a timeout of the agent that was not started.
9386            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9387                Ok(p) => p,
9388                Err(_) => {
9389                    return (
9390                        i,
9391                        job.seat,
9392                        AgentOutcome::ResourceBlocked(format!(
9393                            "no parallel slot became free within {}s",
9394                            slot_limit.as_secs()
9395                        )),
9396                        false,
9397                    );
9398                }
9399            };
9400            spawned_tx.send(job.seat.key.clone()).ok();
9401            let mut seat = job.seat;
9402            let out = repository_wait
9403                .scope(agent::invoke(
9404                    &job.spec,
9405                    &mut seat,
9406                    &Invocation {
9407                        scope: crate::agent::InvocationScope::Repository,
9408                        coordination: coordination.as_ref(),
9409                        fetched_base: fetched_base.as_ref(),
9410                        cwd: &job.cwd,
9411                        prompt: &job.prompt,
9412                        timeout: job.timeout,
9413                        allow_write: job.allow_write,
9414                        unsandboxed: false,
9415                        sessions: job.sessions,
9416                        artifacts: &job.artifacts,
9417                        stem: &job.stem,
9418                        run: &run,
9419                        node: &node,
9420                        cache_dir: cache.as_deref(),
9421                        attachments: &attachments,
9422                        writable: &[],
9423                    },
9424                ))
9425                .await;
9426            let interrupted = out
9427                .as_ref()
9428                .err()
9429                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9430            let out = match out {
9431                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9432                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9433                // Billed work the CLI failed to hand over is not an ordinary
9434                // failure, but its text is the CLI's raw error JSON, not an
9435                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9436                // read it as one by forgetting to check. `usable()` is always
9437                // false here (dropped implies an empty response), so this has
9438                // to be checked before the catch-all `Failed` below or the
9439                // one shape this exists for is lost with the rest.
9440                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9441                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9442                Ok(o) => AgentOutcome::Failed(format!(
9443                    "process exited with no usable output ({})",
9444                    crate::notices::exit_status(o.exit_code)
9445                )),
9446                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9447                    AgentOutcome::ResourceBlocked(e.to_string())
9448                }
9449                Err(e) => AgentOutcome::Failed(e.to_string()),
9450            };
9451            (i, seat, out, interrupted)
9452        });
9453    }
9454    drop(spawned_tx);
9455    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9456    loop {
9457        let joined = tokio::select! {
9458            Some(key) = spawned_rx.recv() => {
9459                // The clock of this seat starts now, not when it queued.
9460                state.seat_spawned(&key);
9461                if let Err(e) = state.save() {
9462                    tracing::warn!("could not persist a seat's start: {e:#}");
9463                }
9464                continue;
9465            }
9466            joined = set.join_next() => joined,
9467        };
9468        let Some(joined) = joined else {
9469            break;
9470        };
9471        let (i, seat, out, interrupted) = match joined {
9472            Ok(v) => v,
9473            // No seat to clear: a panicked task never reported which one it
9474            // was. The defensive sweep below this loop is what stops that
9475            // seat's `active` entry from surviving forever.
9476            Err(e) => {
9477                tracing::error!("agent task panicked: {e}");
9478                continue;
9479            }
9480        };
9481        state.seat_finished(&seat.key);
9482        if interrupted
9483            && node == "implement"
9484            && let Some(label) = state
9485                .candidates
9486                .iter()
9487                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9488                .map(|candidate| candidate.label)
9489        {
9490            state.candidate_preservation.insert(
9491                label,
9492                "interrupted implementation awaiting persistence".into(),
9493            );
9494            state.interrupted_implementations.insert(label);
9495        }
9496        if let AgentOutcome::ResourceBlocked(why) = &out
9497            && state.resource_block.is_none()
9498        {
9499            state.resource_block = Some(crate::run::ResourceBlock {
9500                node: node.to_owned(),
9501                kind: if why.contains("disk pressure") {
9502                    "disk pressure"
9503                } else if why.contains("repository coordination") {
9504                    "repository coordination"
9505                } else {
9506                    "parallel slot"
9507                }
9508                .to_owned(),
9509                detail: why.clone(),
9510                at: Timestamp::now(),
9511            });
9512            let activity = if interrupted {
9513                "agent execution interrupted"
9514            } else {
9515                "no agent was started"
9516            };
9517            state.event("resource", format!("{node}: {activity} - {why}"));
9518        }
9519        record_jobs(state, node, round, &seat.key, &out);
9520        if let Err(e) = state.save() {
9521            tracing::warn!("could not persist a seat's completion: {e:#}");
9522        }
9523        if collected.len() <= i {
9524            collected.resize_with(i + 1, || None);
9525        }
9526        collected[i] = Some((i, seat, out));
9527    }
9528    // Belt-and-braces for the panic branch above: every seat this exact batch
9529    // started shares this `(node, attempt)` pair, and every seat that finished
9530    // normally already cleared itself, so anything left tagged with it here
9531    // can only be a panicked task's leftover. Cleared unconditionally rather
9532    // than left to read as still answering forever.
9533    if state
9534        .active
9535        .values()
9536        .any(|a| a.node == node && a.attempt == attempt)
9537    {
9538        state
9539            .active
9540            .retain(|_, a| !(a.node == node && a.attempt == attempt));
9541        if let Err(e) = state.save() {
9542            tracing::warn!("could not persist the end of a wave: {e:#}");
9543        }
9544    }
9545    // Whether or not the lease above was actually held, several worktrees
9546    // may just have built into the cache with nothing here able to name one
9547    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
9548    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
9549    // clean it might not have strictly needed; trusting a stale match would
9550    // cost it a wrong answer.
9551    if let Some(cache_dir) = cache
9552        && jobs_had_a_writer
9553    {
9554        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9555    }
9556    if let Some(guard) = cache_guard {
9557        guard.release();
9558    }
9559    collected.into_iter().flatten().collect()
9560}
9561
9562/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
9563/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
9564/// this is data collection, not the fix-specific completion contract in
9565/// [`Runner::continue_fix_report`], and applies regardless of which node
9566/// asked.
9567///
9568/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
9569/// evidence from; `Failed` does not, and correctly contributes nothing — a
9570/// timeout or crash is not itself evidence about a command the seat may have
9571/// started.
9572fn record_jobs(
9573    state: &mut RunState,
9574    node: &str,
9575    round: Option<usize>,
9576    seat: &str,
9577    out: &AgentOutcome,
9578) {
9579    let commands: &[agent::CommandEvidence] = match out {
9580        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9581        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9582    };
9583    let checked_at = Timestamp::now();
9584    for c in commands {
9585        state.jobs.push(JobRecord {
9586            node: node.to_owned(),
9587            round,
9588            seat: seat.to_owned(),
9589            id: c.id.clone(),
9590            description: c.description.clone(),
9591            checked_at,
9592            status: match c.exit_code {
9593                Some(0) => JobStatus::Completed,
9594                Some(_) => JobStatus::Failed,
9595                None => JobStatus::Unknown,
9596            },
9597            exit_code: c.exit_code,
9598            result_summary: c.result_summary.clone(),
9599            source: c.source.clone(),
9600        });
9601    }
9602}
9603
9604/// Is a review round clean, given how many reviewer seats answered against
9605/// how many the round expected?
9606///
9607/// A seat that never answered (timeout, crash, unparsable output) is not a
9608/// seat that read the patch and found nothing — treating it as such is
9609/// exactly the bug this function exists to close. Under the default `block`
9610/// policy a missing seat can never be clean; `warn` still requires the seats
9611/// that *did* answer to have found nothing blocking and verification to be
9612/// green.
9613///
9614/// `quota_missing` narrows that `block` default for exactly one cause of
9615/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
9616/// a session limit lifts by the very next round buys nothing — the seat is
9617/// asked again with the same quota — so once every missing seat is accounted
9618/// for by a quota loss (and at least one seat *did* answer, so a decision has
9619/// something to rest on) the round is decided on the panel that could answer,
9620/// same as `warn` would. A panel that lost every seat to quota is not
9621/// decided here: `answered == 0` falls through to the existing `block`
9622/// fallback so a fully collapsed panel still waits rather than landing on no
9623/// review at all.
9624fn round_is_clean(
9625    blocking: usize,
9626    e2e_ok: bool,
9627    answered: usize,
9628    expected: usize,
9629    quota_missing: usize,
9630    policy: IncompleteReviewPolicy,
9631) -> bool {
9632    if blocking != 0 || !e2e_ok {
9633        return false;
9634    }
9635    if answered == expected || policy == IncompleteReviewPolicy::Warn {
9636        return true;
9637    }
9638    answered > 0 && expected - answered <= quota_missing
9639}
9640
9641/// The review loop's own conclusion, derived entirely from its persisted
9642/// round records and the round budget that produced them — never from
9643/// `status`, so a reentry (or `gate`/`merge` reading it independently)
9644/// recomputes the identical answer regardless of what an earlier node in the
9645/// same walk, or a previous walk, did to `status`.
9646///
9647/// `None` while more rounds remain to try, including when review never ran
9648/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
9649/// gone clean, or the budget is spent, or the tree has stopped moving (see
9650/// [`STAGNANT_LIMIT`]), the answer is one of two things:
9651///
9652/// - An incomplete panel that raised nothing is missing input, not a
9653///   verified tree — never a hand-off candidate, whatever verification said
9654///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
9655/// - Otherwise, green e2e on the last round hands off (see
9656///   [`Runner::stop_reviewing`]); red e2e blocks.
9657///
9658/// A last round whose own verification is still `ResourceBlocked` — magi
9659/// itself never got a command to run, not evidence the patch is broken —
9660/// is neither: this returns `None` for it too, the same as "more rounds
9661/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
9662/// handling of that shape) instead of this cheap recomputation guessing a
9663/// verdict a real attempt never produced.
9664pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9665    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9666        return Some(RunStatus::Gating);
9667    }
9668    let last = reviews.last()?;
9669    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9670    if reviews.len() < max_rounds && !stagnant {
9671        return None;
9672    }
9673    if last.incomplete() && last.blocking == 0 {
9674        return Some(RunStatus::Blocked);
9675    }
9676    if last.e2e_status() == E2eStatus::ResourceBlocked {
9677        return None;
9678    }
9679    Some(if last.e2e.iter().all(CommandOutcome::ok) {
9680        RunStatus::Gating
9681    } else {
9682        RunStatus::Blocked
9683    })
9684}
9685
9686/// How long a re-ask may take, given the budget the first attempt had.
9687///
9688/// A `nudged` retry is a request to restate an answer the seat has already
9689/// worked out: it carries no new work, so it does not deserve the original
9690/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
9691/// 133 seconds while a third sat for over ten minutes on a resumed session
9692/// holding 410 KB of prior output - and because the retry had inherited the
9693/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
9694/// judging round whose other seats were long finished.
9695///
9696/// A quarter of the budget, with a floor so that a deliberately short timeout
9697/// does not collapse to nothing. A retry that re-sends the whole prompt
9698/// (because the seat kept no context) is the original job again, and keeps the
9699/// original budget.
9700pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9701    if nudged {
9702        (full / 4).max(Duration::from_secs(120)).min(full)
9703    } else {
9704        full
9705    }
9706}
9707
9708/// Run a wave and parse each reply, re-asking the seats whose reply was
9709/// unusable.
9710///
9711/// The re-ask is a nudge rather than the whole prompt again when the seat still
9712/// holds its conversation, which is the difference between a cheap retry and
9713/// paying for the entire candidate set twice.
9714///
9715/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
9716/// successor in `roster` is handed to it instead of being re-asked: the
9717/// handover is the retry. A seat with no successor left (a single-agent
9718/// roster, the roster's tail) is nudged as before, up to `retries` times. So
9719/// the asks to one seat in one node number at most
9720/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
9721/// is asked once (a dropped stream is nudged first), and only the last agent
9722/// of the chain gets the `retries` same-agent nudges. Each roster agent is
9723/// tried at most once per seat, walking forward from the seat's own position and never wrapping
9724/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
9725/// other failure stops the chain when the previous agent failed the same way
9726/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
9727/// [`has_context`] is false and the job's own full prompt and full budget are
9728/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
9729/// intermediate ones are not losses) and is returned as a failure like any
9730/// other absent seat — the caller decides whether the panel still has a
9731/// quorum. An empty `roster` disables handover: failures are nudged as they
9732/// always were, and a quota is simply lost. A reply that fails to parse or
9733/// validate is the prompt's doing and is only ever nudged, never handed over.
9734///
9735/// The returned [`SeatState`] names the agent that answered (or tried last).
9736#[allow(clippy::too_many_arguments)]
9737async fn ask_json_wave<T>(
9738    jobs: Vec<SeatJob>,
9739    sem: Arc<Semaphore>,
9740    retries: usize,
9741    roster: &[AgentSpec],
9742    ctx: &WaveCtx<'_>,
9743    losses: &mut Vec<QuotaLoss>,
9744    state: &mut RunState,
9745    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9746) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9747where
9748    T: serde::de::DeserializeOwned + Send + 'static,
9749{
9750    ask_wave_with(
9751        jobs,
9752        sem,
9753        retries,
9754        roster,
9755        ctx,
9756        losses,
9757        state,
9758        &|text: &str| {
9759            let v = verdict::extract_json::<T>(text)?;
9760            validate(&v)?;
9761            Ok(v)
9762        },
9763    )
9764    .await
9765}
9766
9767/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
9768/// a node whose answer is prose (deliberation) shares the same handover,
9769/// failure classification, quota bookkeeping and bounds instead of a copy.
9770///
9771/// A seat handed to another roster agent is sent the job's `handover` prompt
9772/// (when it has one) rather than `prompt`: the new agent has no session, so a
9773/// resume-style prompt would be incomplete. That holds for the handover ask
9774/// and for every nudge to that agent whose `has_context` is false.
9775#[allow(clippy::too_many_arguments)]
9776async fn ask_wave_with<T>(
9777    jobs: Vec<SeatJob>,
9778    sem: Arc<Semaphore>,
9779    retries: usize,
9780    roster: &[AgentSpec],
9781    ctx: &WaveCtx<'_>,
9782    losses: &mut Vec<QuotaLoss>,
9783    state: &mut RunState,
9784    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9785) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9786where
9787    T: Send + 'static,
9788{
9789    let n = jobs.len();
9790    let originals: Vec<SeatJob> = jobs;
9791    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9792    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9793    // Nudges each seat's *current* agent has taken — 0 for a first-ask
9794    // answer, N once it has gone through N nudges. Read back once this
9795    // returns, so a caller building a history record (`ReviewRecord`) can
9796    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
9797    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
9798    // field's own doc.
9799    let mut nudges: Vec<usize> = vec![0; n];
9800    // The agent now occupying each seat, the ids it has already been through,
9801    // where in the roster the walk began, the class of the last failure, and
9802    // the stem word of a handover not yet asked (full prompt, full budget).
9803    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9804    let mut tried: Vec<BTreeSet<String>> = specs
9805        .iter()
9806        .map(|s| BTreeSet::from([s.id.clone()]))
9807        .collect();
9808    let starts: Vec<usize> = specs
9809        .iter()
9810        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9811        .collect();
9812    let carry = ctx.carry_seats && !roster.is_empty();
9813    // Carried across rounds: ids that failed the seat earlier, and how the
9814    // last failure went (so a repeat of it is not handed over again).
9815    let carried: Vec<BTreeSet<String>> = originals
9816        .iter()
9817        .map(|j| {
9818            state
9819                .seat_history
9820                .get(&j.seat.key)
9821                .filter(|_| carry)
9822                .map(|h| h.failed.clone())
9823                .unwrap_or_default()
9824        })
9825        .collect();
9826    let mut prev: Vec<Option<FailClass>> = originals
9827        .iter()
9828        .map(|j| {
9829            state
9830                .seat_history
9831                .get(&j.seat.key)
9832                .filter(|_| carry)
9833                .and_then(|h| h.last_fail.clone())
9834        })
9835        .collect();
9836    let next_agent =
9837        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9838            let others: BTreeSet<String> = specs
9839                .iter()
9840                .enumerate()
9841                .filter(|(j, _)| *j != i)
9842                .map(|(_, s)| s.id.clone())
9843                .collect();
9844            pick_successor(
9845                roster,
9846                starts[i],
9847                tried,
9848                carry.then(|| &carried[i]),
9849                &others,
9850            )
9851            .cloned()
9852        };
9853    let mut fresh: Vec<Option<String>> = vec![None; n];
9854    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9855    let mut pending: Vec<usize> = (0..n).collect();
9856
9857    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
9858    // asks: an agent with a successor is asked once and handed over, and only
9859    // a seat with no successor spends `retries` nudges on the same agent. This
9860    // only guarantees the loop's own termination whatever those say.
9861    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9862    for round in 0..max_rounds {
9863        if pending.is_empty() {
9864            break;
9865        }
9866        let mut batch = Vec::with_capacity(pending.len());
9867        let mut renudged: Vec<&str> = Vec::new();
9868        for &i in &pending {
9869            let src = &originals[i];
9870            // A seat now held by another agent than the job named has no
9871            // session of its own: it gets the full-context prompt whenever
9872            // it is asked in full (the handover ask, a nudge it cannot
9873            // resume).
9874            let full: &str = match &src.handover {
9875                Some(h) if specs[i].id != src.spec.id => h,
9876                _ => &src.prompt,
9877            };
9878            // The prompt and the budget are one decision: a nudge restates
9879            // finished work, a re-sent prompt redoes it.
9880            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9881                (
9882                    full.to_owned(),
9883                    src.timeout,
9884                    format!("{}-{word}-{}", src.stem, specs[i].id),
9885                )
9886            } else if nudges[i] == 0 {
9887                (full.to_owned(), src.timeout, src.stem.clone())
9888            } else {
9889                renudged.push(src.seat.key.as_str());
9890                let why = done[i]
9891                    .as_ref()
9892                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
9893                    .unwrap_or_else(|| "no parsable answer".to_owned());
9894                let nudge = prompt::nudge(&why);
9895                let nudged = has_context(&specs[i], &seats[i], src.sessions);
9896                let prompt = if nudged {
9897                    nudge
9898                } else {
9899                    format!("{full}\n\n---\n\n{nudge}")
9900                };
9901                (
9902                    prompt,
9903                    retry_budget(src.timeout, nudged),
9904                    format!("{}-retry{}", src.stem, nudges[i]),
9905                )
9906            };
9907            batch.push(SeatJob {
9908                spec: specs[i].clone(),
9909                seat: seats[i].clone(),
9910                cwd: src.cwd.clone(),
9911                prompt,
9912                timeout,
9913                allow_write: src.allow_write,
9914                sessions: src.sessions,
9915                artifacts: src.artifacts.clone(),
9916                stem,
9917                handover: None,
9918            });
9919        }
9920
9921        if !renudged.is_empty() {
9922            state.event(
9923                ctx.node,
9924                format!("retry {round}: re-asking {}", renudged.join(", ")),
9925            );
9926        }
9927        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9928        let mut still = Vec::new();
9929        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9930            seats[i] = seat;
9931            // No agent was asked: nothing to hand over, nudge, remember or
9932            // charge. The caller sees `resource_block` and stops the node.
9933            if let AgentOutcome::ResourceBlocked(why) = &out {
9934                done[i] = Some(Err(anyhow::anyhow!("{why}")));
9935                continue;
9936            }
9937            let class = FailClass::of(&out);
9938            // A dropped stream is nudged first (the conversation is still
9939            // there to pick up); only a seat whose nudges are spent hands over.
9940            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9941                && nudges[i] < retries
9942                && !roster.is_empty();
9943            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9944                let next = should_hand_over(prev[i].as_ref(), &cur)
9945                    .then(|| next_agent(i, &tried[i], &specs))
9946                    .flatten();
9947                if carry {
9948                    let h = state
9949                        .seat_history
9950                        .entry(originals[i].seat.key.clone())
9951                        .or_default();
9952                    h.failed.insert(specs[i].id.clone());
9953                    h.last_fail = Some(cur.clone());
9954                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9955                        h.last_ok = None;
9956                    }
9957                    // Saved before the next agent is asked, so a restart in
9958                    // between does not forget who failed.
9959                    if let Err(e) = state.save() {
9960                        tracing::warn!("could not persist a seat's failure history: {e:#}");
9961                    }
9962                }
9963                if let Some(next) = next {
9964                    record_handover(
9965                        state,
9966                        ctx.node,
9967                        &originals[i].seat.key,
9968                        &specs[i].id,
9969                        &next.id,
9970                        &cur,
9971                        &fail_reason(&out),
9972                    );
9973                    tried[i].insert(next.id.clone());
9974                    prev[i] = Some(cur.clone());
9975                    seats[i] =
9976                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9977                    specs[i] = next;
9978                    fresh[i] = Some(cur.stem_word().to_owned());
9979                    nudges[i] = 0;
9980                    done[i] = Some(Err(anyhow::anyhow!(
9981                        "handed over after: {}",
9982                        fail_reason(&out)
9983                    )));
9984                    still.push(i);
9985                    continue;
9986                }
9987            }
9988            if carry
9989                && !nudge_first
9990                && let Some(cur) = class.clone()
9991            {
9992                // The chain ended here (no successor, or a repeated failure).
9993                let h = state
9994                    .seat_history
9995                    .entry(originals[i].seat.key.clone())
9996                    .or_default();
9997                h.failed.insert(specs[i].id.clone());
9998                h.last_fail = Some(cur);
9999            }
10000            let parsed = match out {
10001                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10002                AgentOutcome::Quota(o) => {
10003                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10004                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10005                }
10006                // Not a parseable answer: the nudge loop re-asks it, which is
10007                // exactly what a dropped stream needs. Just don't hand its raw
10008                // error JSON to `extract_json`.
10009                AgentOutcome::Dropped(o) => {
10010                    let why = o
10011                        .dropped
10012                        .as_ref()
10013                        .map(|d| d.why.as_str())
10014                        .unwrap_or("the CLI ended the stream without delivering its answer");
10015                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10016                }
10017                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10018                    Err(anyhow::anyhow!(e))
10019                }
10020            };
10021            let quota = class == Some(FailClass::Quota);
10022            let failed = parsed.is_err();
10023            done[i] = Some(parsed);
10024            if carry && !failed {
10025                let h = state
10026                    .seat_history
10027                    .entry(originals[i].seat.key.clone())
10028                    .or_default();
10029                h.failed.remove(&specs[i].id);
10030                h.last_ok = Some(specs[i].id.clone());
10031                h.last_fail = None;
10032            }
10033            // Do not re-ask a rate-limited seat (quota) — a retry is known to
10034            // fail the same way; and never re-ask a seat that already parsed.
10035            // A failed agent that still has a successor is not re-asked
10036            // either: the handover was its remedy and has just been refused
10037            // (the chain stops on a repeated failure class). A seat with no
10038            // successor left (a single-agent roster, the roster's tail, or an
10039            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10040            let agent_failure = class.is_some()
10041                && !nudge_first
10042                && !roster.is_empty()
10043                && next_agent(i, &tried[i], &specs).is_some();
10044            if failed && !quota && !agent_failure && nudges[i] < retries {
10045                nudges[i] += 1;
10046                still.push(i);
10047            }
10048        }
10049        pending = still;
10050    }
10051
10052    // One loss per seat whose chain ended on a quota: the intermediate ones
10053    // were absorbed by a handover and are not losses.
10054    for (i, q) in last_quota.into_iter().enumerate() {
10055        if let Some(reset) = q {
10056            losses.push(QuotaLoss {
10057                seat: originals[i].seat.key.clone(),
10058                node: ctx.node.to_owned(),
10059                at: Timestamp::now(),
10060                reset,
10061            });
10062        }
10063    }
10064
10065    seats
10066        .into_iter()
10067        .zip(done)
10068        .zip(nudges)
10069        .map(|((seat, res), attempts)| {
10070            (
10071                seat,
10072                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10073                attempts,
10074            )
10075        })
10076        .collect()
10077}
10078
10079/// Did the wave end without asking any agent (see [`block_wave`])?
10080fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10081    results
10082        .iter()
10083        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10084}
10085
10086/// Settle a wave that never spawned anything: every job answers
10087/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10088/// else about the seat (session, history, quota) is touched.
10089fn block_wave(
10090    jobs: Vec<SeatJob>,
10091    state: &mut RunState,
10092    node: &str,
10093    attempt: usize,
10094    why: &str,
10095) -> Vec<(usize, SeatState, AgentOutcome)> {
10096    state.resource_wait = None;
10097    state
10098        .active
10099        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10100    if let Err(e) = state.save() {
10101        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10102    }
10103    jobs.into_iter()
10104        .enumerate()
10105        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10106        .collect()
10107}
10108
10109/// Acquire the shared build cache's lease, waiting out contention within
10110/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10111/// unbounded wait is never acceptable).
10112///
10113/// A first, non-blocking check happens before ever waiting; if it finds the
10114/// lease busy, that fact is logged as a `verify` event *and* flushed with
10115/// [`RunState::save`] immediately — not only once the wait finally succeeds
10116/// or gives up — so a `magi show` run by a different process while this one
10117/// is still waiting reads a `run.json` that says so, rather than whatever it
10118/// looked like before the wait started. The same applies to the terminal
10119/// failure: logged and saved before this returns `Err`, so a caller that
10120/// could not get the lease at all still leaves a legible record of why.
10121async fn acquire_cache_lease(
10122    state: &mut RunState,
10123    cache_dir: &Path,
10124    owner: &crate::cache::Owner,
10125    budget: Duration,
10126    context: &str,
10127    cancel: &(dyn Fn() -> bool + Sync),
10128) -> Result<Option<crate::cache::Guard>> {
10129    let home = crate::run::home();
10130    let started = Instant::now();
10131    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10132        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10133        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10134        Err(e) => {
10135            state.event(
10136                "verify",
10137                format!("{context}: could not check the shared build cache: {e:#}"),
10138            );
10139            if let Err(e2) = state.save() {
10140                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10141            }
10142            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10143        }
10144    };
10145    state.event(
10146        "verify",
10147        format!(
10148            "{context}: waiting for the shared build cache at {} ({})",
10149            cache_dir.display(),
10150            busy.describe()
10151        ),
10152    );
10153    // Seats of this node are waiting, not running: say so, so the countdown a
10154    // reader sees only ever measures an agent that has been spawned.
10155    state.resource_wait = Some(crate::run::ResourceWait {
10156        node: context.to_owned(),
10157        kind: "shared build cache".to_owned(),
10158        holder: busy.describe(),
10159        since: Timestamp::now(),
10160        limit_secs: budget.as_secs(),
10161    });
10162    if let Err(e) = state.save() {
10163        tracing::warn!("could not persist a cache wait: {e:#}");
10164    }
10165    let remaining = budget.saturating_sub(started.elapsed());
10166    let waited = crate::cache::wait_for_unless(
10167        &home,
10168        cache_dir,
10169        owner,
10170        remaining,
10171        Duration::from_secs(5),
10172        cancel,
10173    )
10174    .await;
10175    state.resource_wait = None;
10176    match waited {
10177        Ok(g) => Ok(g),
10178        Err(e) => {
10179            state.event("verify", format!("{context}: {e:#}"));
10180            if let Err(e2) = state.save() {
10181                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10182            }
10183            Err(e)
10184        }
10185    }
10186}
10187
10188/// The lease state could not be read at all (unreadable lease file, a
10189/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10190/// known to be contended, so a wave degrades to running without a shared cache
10191/// instead of stopping on a hold that [`crate::triage`] could never release.
10192#[derive(Debug)]
10193struct CacheCheckError(String);
10194
10195impl std::fmt::Display for CacheCheckError {
10196    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10197        f.write_str(&self.0)
10198    }
10199}
10200
10201impl std::error::Error for CacheCheckError {}
10202
10203/// Run `body` — a verify command batch — while holding the shared build
10204/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10205/// never interleave with another borrower's build against the same
10206/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10207/// timeout, or a human's own `magi review`. See the `cache` module doc for
10208/// why this matters more than Cargo's own per-target locking covers — two
10209/// *different* worktrees building the same package name/version into one
10210/// cache directory is a staleness bug, not a lock contention one.
10211///
10212/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10213/// agent waves do. Repository scheduling also has its own bounded,
10214/// cancellable resource-wait policy. Contention must not consume the
10215/// verification budget: `body` receives the full `budget` after both leases
10216/// are acquired and the cache is fresh. Without a shared cache, only
10217/// repository coordination is acquired before handing over the full budget.
10218///
10219/// A lease that cannot be acquired within the resource wait is reported as a single
10220/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10221/// verification — the same shape a spawn failure already takes in
10222/// [`run_commands`], so a caller need not special-case it.
10223#[allow(clippy::too_many_arguments)]
10224async fn with_cache_lease<'s, F, Fut>(
10225    state: &'s mut RunState,
10226    cache_dir: Option<&Path>,
10227    node: &str,
10228    seat: &str,
10229    worktree: &Path,
10230    head: &str,
10231    budget: Duration,
10232    context: &str,
10233    wait: &crate::repo_lock::WaitPolicy,
10234    body: F,
10235) -> Result<(Vec<CommandOutcome>, bool)>
10236where
10237    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10238    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10239{
10240    // Match editing waves: cache first, repository second. In particular,
10241    // pre_gate must never hold the repository while waiting for a wave's cache.
10242    let started = Instant::now();
10243    let home = crate::run::home();
10244    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10245    let guard = if let Some(cache_dir) = cache_dir {
10246        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10247        match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10248            wait.cancelled()
10249        })
10250        .await
10251        .and_then(|g| {
10252            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10253        }) {
10254            Ok(g) => Some(g),
10255            Err(e) => {
10256                return Ok((
10257                    vec![CommandOutcome {
10258                        command: "(waiting for the shared build cache)".into(),
10259                        code: None,
10260                        output_tail: e.to_string(),
10261                        duration_ms: started.elapsed().as_millis() as u64,
10262                        resource_blocked: true,
10263                    }],
10264                    false,
10265                ));
10266            }
10267        }
10268    } else {
10269        None
10270    };
10271    let _coordination = coordinate_operation(state, worktree, wait).await?;
10272    if let Some(cache_dir) = cache_dir {
10273        let identity = crate::cache::Identity::new(worktree, head);
10274        if let Err(e) = crate::cache::ensure_fresh(
10275            &home,
10276            cache_dir,
10277            &identity,
10278            &state.config.verify.cargo_manifests(),
10279        ) {
10280            state.event(
10281                "verify",
10282                format!(
10283                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10284                    worktree.display(),
10285                    short(head)
10286                ),
10287            );
10288            if let Some(guard) = guard {
10289                guard.release();
10290            }
10291            return Ok((
10292                vec![CommandOutcome {
10293                    command: "(confirming the shared build cache is fresh)".into(),
10294                    code: None,
10295                    output_tail: e.to_string(),
10296                    duration_ms: started.elapsed().as_millis() as u64,
10297                    resource_blocked: true,
10298                }],
10299                false,
10300            ));
10301        }
10302    }
10303    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10304    if !timed_out_pids.is_empty() {
10305        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10306    }
10307    // Drop repository coordination before releasing the cache, reversing the
10308    // acquisition order. A cancelled/error future also drops both guards.
10309    drop(_coordination);
10310    if let Some(guard) = guard {
10311        guard.release();
10312    }
10313    Ok((outcomes, retried))
10314}
10315
10316/// Poll `pids` — commands [`run_commands`] reports as still running when its
10317/// own timeout elapsed — until every one is confirmed gone, or
10318/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10319///
10320/// Real confirmation where confirmation is possible, not a substitute for
10321/// full process-tree observation: a grandchild the timed-out process spawned
10322/// and that survives independently of it is invisible to a pid check the
10323/// same way it always was, and continuing to observe and collect *that*
10324/// stays a different piece of work with its own owner. This only narrows a
10325/// fixed blind wait into an actual check of the pids this process does know
10326/// about.
10327async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10328    wait_for_pids_with(
10329        pids,
10330        crate::proc::pid_alive,
10331        LEASE_RELEASE_POLL,
10332        LEASE_RELEASE_MAX_WAIT,
10333    )
10334    .await;
10335}
10336
10337/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10338/// interval and ceiling supplied by the caller, so the polling *logic* -
10339/// returns as soon as every pid reports dead, gives up at the ceiling
10340/// otherwise - is testable on millisecond durations without asking the real
10341/// OS about a pid at all.
10342async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10343    pids: &[u32],
10344    alive: F,
10345    poll: Duration,
10346    max_wait: Duration,
10347) {
10348    let deadline = Instant::now() + max_wait;
10349    loop {
10350        if pids.iter().all(|&pid| !alive(pid)) {
10351            return;
10352        }
10353        if Instant::now() >= deadline {
10354            return;
10355        }
10356        tokio::time::sleep(poll).await;
10357    }
10358}
10359
10360/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10361/// admission that it could not even get a verify command to run, as opposed
10362/// to evidence the command actually produced? A caller that would otherwise
10363/// read a resource-blocked outcome as a red command must check this first:
10364/// see [`Runner::gate`], which retries rather than records `Blocked` when
10365/// this is true.
10366fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10367    outcomes.iter().any(|o| o.resource_blocked)
10368}
10369
10370/// What [`Runner::gate_fix_round`] decided.
10371pub(crate) enum GateFix {
10372    /// The tree changed and `verify.e2e` is still green: run the gate again.
10373    Retry,
10374    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10375    /// last failure stands and the run ends blocked.
10376    Stop,
10377    /// `verify.e2e` could not run after the fix (magi's own contention):
10378    /// decide nothing now, a later reentry retries.
10379    Defer,
10380}
10381
10382/// Is every red command in `outcomes` an ordinary failure the code could
10383/// explain: it ran, exited non-zero, and said something?
10384///
10385/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10386/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10387/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10388/// what the command is stays the gate's business.
10389fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10390    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10391    red.peek().is_some()
10392        && red.all(|o| {
10393            !o.resource_blocked
10394                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10395                && !o.output_tail.trim().is_empty()
10396        })
10397}
10398
10399/// Describe one verify command's outcome for the event log, distinguishing a
10400/// build/link failure — the toolchain never produced a binary to run — from
10401/// an actual test failure, since only the latter is a verdict on the patch.
10402fn e2e_outcome_label(o: &CommandOutcome) -> String {
10403    if o.ok() {
10404        return "pass".to_owned();
10405    }
10406    let reason = if o.build_failed() {
10407        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10408    } else {
10409        format!("FAIL ({:?})", o.code)
10410    };
10411    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10412}
10413
10414/// Run `verify.e2e`, retrying once if the first attempt could not build or
10415/// link — a build/link failure is frequently a race against a shared
10416/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
10417/// `verify` event per command, tagged with `context` (normally `"round N"`)
10418/// so the two call sites that need this — the ordinary per-round leg in
10419/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
10420/// it will ever call a round green — read identically in the event log.
10421async fn run_e2e_with_retry(
10422    state: &mut RunState,
10423    shell: &[String],
10424    commands: &[String],
10425    worktree: &Path,
10426    timeout: Duration,
10427    context: &str,
10428) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10429    let (mut e2e, mut timed_out_pids) = run_commands(
10430        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10431    )
10432    .await;
10433    for o in &e2e {
10434        state.event(
10435            "verify",
10436            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10437        );
10438    }
10439    // A build/link failure is not a verdict on the patch — it is frequently a
10440    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
10441    // one retry before letting a red like that decide the round.
10442    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10443    if verify_retried {
10444        state.event(
10445            "verify",
10446            format!(
10447                "{context}: verify could not build/link, not a test result — retrying once \
10448                 before concluding"
10449            ),
10450        );
10451        let retried = run_commands(
10452            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10453        )
10454        .await;
10455        e2e = retried.0;
10456        // Both attempts' timeouts matter, not just the last one: the first
10457        // attempt's descendants may still be alive alongside the retry's.
10458        timed_out_pids.extend(retried.1);
10459        for o in &e2e {
10460            state.event(
10461                "verify",
10462                format!(
10463                    "{context}: retry `{}` -> {}",
10464                    o.command,
10465                    e2e_outcome_label(o)
10466                ),
10467            );
10468        }
10469    }
10470    (e2e, verify_retried, timed_out_pids)
10471}
10472
10473/// Run configured shell commands in `cwd`, in order. The second element is
10474/// the pid of every command that hit `timeout` and was still running when
10475/// this stopped waiting on it (best-effort: `None` when the platform did not
10476/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
10477/// that releases a shared resource afterward needs to know.
10478///
10479/// Records `task` into [`RunState::active`] at every command boundary
10480/// (`RunState::task_command`) and clears it once the whole list has run
10481/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
10482/// for minutes with no seat and no output of its own to show for it (see
10483/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
10484/// apart from "not yet run" without this), and this is the only place that
10485/// knows which command is running right now and how many are left. Three
10486/// saves per command — start, not per second — matching the same "only at a
10487/// boundary" rule [`wave`] already follows for seats.
10488#[allow(clippy::too_many_arguments)]
10489async fn run_commands(
10490    state: &mut RunState,
10491    node: &str,
10492    task: &str,
10493    attempt: usize,
10494    shell: &[String],
10495    commands: &[String],
10496    cwd: &Path,
10497    timeout: Duration,
10498) -> (Vec<CommandOutcome>, Vec<u32>) {
10499    if commands.is_empty() {
10500        // Nothing to mark as running and nothing to clear — an empty list
10501        // means "not configured", and touching `active` (or the disk) over
10502        // that would be a write for every round of a repo with no
10503        // `verify.e2e` / `verify.gate` commands at all.
10504        return (Vec::new(), Vec::new());
10505    }
10506    let mut out = Vec::new();
10507    let mut timed_out_pids = Vec::new();
10508    let total = commands.len();
10509    for (idx, command) in commands.iter().enumerate() {
10510        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10511        if let Err(e) = state.save() {
10512            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10513        }
10514        let started = Instant::now();
10515        let mut cmd = tokio::process::Command::new(&shell[0]);
10516        cmd.quiet();
10517        cmd.args(&shell[1..])
10518            .arg(command)
10519            .current_dir(cwd)
10520            .stdin(std::process::Stdio::null())
10521            .stdout(std::process::Stdio::piped())
10522            .stderr(std::process::Stdio::piped())
10523            .kill_on_drop(true);
10524        let mut outputs = Vec::new();
10525        let protected = (|| -> Result<()> {
10526            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10527                outputs.push(guard);
10528            }
10529            if let Some(cache) = state.config.cache_dir() {
10530                outputs.push(crate::retention::OutputLease::cache(
10531                    &crate::run::home(),
10532                    &cache,
10533                    false,
10534                )?);
10535            }
10536            Ok(())
10537        })();
10538        if let Err(e) = protected {
10539            out.push(CommandOutcome {
10540                command: command.clone(),
10541                code: None,
10542                output_tail: format!("output ownership unavailable: {e:#}"),
10543                duration_ms: 0,
10544                resource_blocked: true,
10545            });
10546            continue;
10547        }
10548        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10549        let (code, body) = match spawned {
10550            Ok(mut running) => {
10551                let pid = running.child().id();
10552                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10553                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10554                let mut paths = vec![cwd.to_owned()];
10555                if let Some(cache) = state.config.cache_dir() {
10556                    paths.push(cache);
10557                }
10558                let mut pressure = None;
10559                let status = tokio::select! {
10560                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10561                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10562                        pressure = Some(reason); None
10563                    }
10564                };
10565                let (code, failure) = match status {
10566                    Some(Ok(Ok(status))) => (
10567                        status.code(),
10568                        status
10569                            .code()
10570                            .is_none()
10571                            .then(|| format!("command terminated: {status}")),
10572                    ),
10573                    Some(Ok(Err(error))) => {
10574                        (None, Some(format!("failed waiting for command: {error}")))
10575                    }
10576                    Some(Err(_)) => {
10577                        // Leased cleanup confirms process-tree death itself.
10578                        // Unleased commands still need the caller's PID check
10579                        // after kill-on-drop, before releasing cache ownership.
10580                        if !running.coordinated()
10581                            && let Some(pid) = pid
10582                        {
10583                            timed_out_pids.push(pid);
10584                        }
10585                        (
10586                            None,
10587                            Some(format!("timed out after {}s", timeout.as_secs())),
10588                        )
10589                    }
10590                    None => (None, None),
10591                };
10592                running.finish().await;
10593                if let Some(reason) = pressure {
10594                    state.resource_block = Some(crate::run::ResourceBlock {
10595                        node: node.to_owned(),
10596                        kind: "disk pressure".to_owned(),
10597                        detail: reason.clone(),
10598                        at: Timestamp::now(),
10599                    });
10600                    out.push(CommandOutcome {
10601                        command: command.clone(),
10602                        code: None,
10603                        output_tail: reason,
10604                        duration_ms: started.elapsed().as_millis() as u64,
10605                        resource_blocked: true,
10606                    });
10607                    break;
10608                }
10609                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10610                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10611                if let Some(failure) = failure {
10612                    body.push('\n');
10613                    body.push_str(&failure);
10614                }
10615                (code, body)
10616            }
10617            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10618        };
10619        out.push(CommandOutcome {
10620            command: command.clone(),
10621            code,
10622            output_tail: tail(&body, OUTPUT_TAIL),
10623            duration_ms: started.elapsed().as_millis() as u64,
10624            resource_blocked: false,
10625        });
10626    }
10627    state.task_finished(task);
10628    if let Err(e) = state.save() {
10629        tracing::warn!("could not persist the end of {task}: {e:#}");
10630    }
10631    (out, timed_out_pids)
10632}
10633
10634/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
10635/// section (`report::run`) and in the `merge` event this node records — for
10636/// the operator to run by hand.
10637///
10638/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
10639/// branch whose ruleset forbids merge commits (GitHub's "must not contain
10640/// merge commits", or "require linear history") rejects the push a `--no-ff`
10641/// merge would produce, which is exactly the guidance this function replaces.
10642/// `message`'s first line becomes the squash commit's subject, matching the
10643/// note `report::run` prints alongside this command — see that function for
10644/// why an explicit subject is not optional there.
10645fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10646    let repo = repo.display();
10647    match style {
10648        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10649        MergeStyle::Squash => {
10650            // The subject sits inside double quotes, and a title an agent
10651            // wrote may carry the characters that break out of them.
10652            let subject = message
10653                .lines()
10654                .next()
10655                .unwrap_or(branch)
10656                .replace(['\\', '"', '$', '`'], "");
10657            format!(
10658                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10659            )
10660        }
10661        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10662    }
10663}
10664
10665/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
10666/// under the hood, rejects a `title` over 256 characters and the whole
10667/// command fails — no PR at all, for a run whose body was otherwise fine
10668/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
10669/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
10670/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
10671/// this bound. It is a margin, not a guarantee — a title packed
10672/// with multi-unit characters could still in principle land close to the
10673/// edge, but a real task title's occasional emoji or accented letter fits
10674/// comfortably inside it.
10675const PR_TITLE_MAX: usize = 240;
10676
10677/// A pull request title from the opening line of `text`, or `None` when that
10678/// line is not English (GitHub text is) or has no letters.
10679///
10680/// A line within `max` is kept as is. A longer one is cut at the end of its
10681/// first sentence when that falls inside `max`, else at a word boundary with a
10682/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
10683/// make every merely-truncated title look non-English). The language check
10684/// runs on the kept text before any mark is added, so only what GitHub will
10685/// show is judged: an English opening followed by non-ASCII far past the cut
10686/// still passes.
10687fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10688    let line = queue::first_line(text)?;
10689    let chars: Vec<char> = line.chars().collect();
10690    let (kept, mark) = if chars.len() <= max {
10691        (line.to_owned(), "")
10692    } else if let Some(end) = sentence_end(&chars, max) {
10693        (chars[..end].iter().collect::<String>(), "")
10694    } else {
10695        let room = max.saturating_sub(3);
10696        // Cut at the last space inside the room; when the char just past the
10697        // room is a space the room already ends on a word.
10698        let cut = if chars[room].is_whitespace() {
10699            room
10700        } else {
10701            chars[..room]
10702                .iter()
10703                .rposition(|c| c.is_whitespace())
10704                .unwrap_or(room)
10705        };
10706        let head: String = chars[..cut].iter().collect();
10707        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10708        (head.to_owned(), "...")
10709    };
10710    if kept.is_empty() {
10711        return None;
10712    }
10713    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10714        return None;
10715    }
10716    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10717        return None;
10718    }
10719    Some(format!("{kept}{mark}"))
10720}
10721
10722/// The char length of the first sentence of `chars` when it ends within `max`
10723/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
10724/// abbreviations so `e.g. foo` does not end a title early.
10725fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10726    const MIN: usize = 20;
10727    for i in MIN..max.min(chars.len()) {
10728        if !matches!(chars[i], '.' | '!' | '?') {
10729            continue;
10730        }
10731        let Some(&next) = chars.get(i + 1) else {
10732            continue;
10733        };
10734        if !next.is_whitespace() {
10735            continue;
10736        }
10737        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10738        if after.is_some_and(|c| c.is_ascii_lowercase()) {
10739            continue;
10740        }
10741        let word: String = chars[..i]
10742            .iter()
10743            .rev()
10744            .take_while(|c| !c.is_whitespace())
10745            .collect::<Vec<_>>()
10746            .into_iter()
10747            .rev()
10748            .collect();
10749        let word = word.to_ascii_lowercase();
10750        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10751            continue;
10752        }
10753        let end = chars[..i]
10754            .iter()
10755            .rposition(|c| !c.is_whitespace())
10756            .map_or(i, |p| p + 1);
10757        return Some(end);
10758    }
10759    None
10760}
10761
10762/// What `merge = "pr"` (and the merge commit of the other modes) says about a
10763/// change: a title and a body describing what was *implemented*, not the task
10764/// that asked for it. A task reads as a request; a reader of the merged
10765/// history wants the change.
10766struct PrMessage {
10767    title: String,
10768    body: String,
10769}
10770
10771impl PrMessage {
10772    /// Title, blank line, body. The first line is the squash/merge commit
10773    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
10774    /// has to stay one sensible line.
10775    fn commit_message(&self) -> String {
10776        format!("{}\n\n{}", self.title, self.body)
10777    }
10778}
10779
10780/// The text after a leading `TITLE:` (any case) on `line`.
10781fn title_marker(line: &str) -> Option<&str> {
10782    let line = line.trim();
10783    let head = line.get(..6)?;
10784    head.eq_ignore_ascii_case("title:")
10785        .then(|| line[6..].trim())
10786}
10787
10788/// The implementer's own one-line title: the `TITLE:` line the implement
10789/// prompt asks for at the top of its SUMMARY. Candidate commits are all
10790/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
10791/// source, and a title that says as much is refused here too.
10792fn summary_title(summary: &str) -> Option<String> {
10793    let first = summary.lines().find(|l| !l.trim().is_empty())?;
10794    let raw = title_marker(first)?;
10795    if raw.is_empty() {
10796        return None;
10797    }
10798    let title = queue::title_from(raw, PR_TITLE_MAX);
10799    let lower = title.to_ascii_lowercase();
10800    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10801        return None;
10802    }
10803    Some(title)
10804}
10805
10806/// How `open_review`'s instruction begins; see [`landing_title`].
10807const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10808
10809/// Marker `open_review` gives a candidate that nothing in the roster wrote.
10810pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10811
10812/// Does this run review work that already existed, rather than implement a
10813/// task? Runs recorded before `reviewed_commits` existed carry only the
10814/// candidate marker.
10815pub(crate) fn is_review_run(state: &RunState) -> bool {
10816    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10817}
10818
10819/// The title of a review-only run: the subject of the oldest commit under
10820/// review. Later commits are usually fixups, and `instruction` is the review
10821/// prompt, which says nothing about the change. GitHub text is English, so a
10822/// non-ASCII or blank subject yields `None` and the caller's neutral title.
10823fn review_title(state: &RunState) -> Option<String> {
10824    english_subject(
10825        state.reviewed_commits.as_ref()?.first()?,
10826        state.config.graph.github_english(),
10827    )
10828}
10829
10830/// `raw` as a pull request title, or `None` when it is blank, not English
10831/// (GitHub text is), or one of magi's own candidate commit subjects.
10832fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10833    let raw = raw.trim();
10834    if raw.is_empty() {
10835        return None;
10836    }
10837    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10838        return None;
10839    }
10840    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10841        return None;
10842    }
10843    let title = queue::title_from(raw, PR_TITLE_MAX);
10844    let lower = title.to_ascii_lowercase();
10845    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10846        return None;
10847    }
10848    Some(title)
10849}
10850
10851/// What a review-only run's branch says about itself, read at the moment the
10852/// pull request is opened.
10853#[derive(Debug, Clone, PartialEq, Eq)]
10854struct BranchFacts {
10855    /// `(subject, body)` of each commit, oldest first.
10856    commits: Vec<(String, String)>,
10857    /// Trimmed `git diff --stat`.
10858    stat: String,
10859}
10860
10861/// Longest diff stat shown: this many file lines plus the summary line.
10862const STAT_FILE_LINES: usize = 25;
10863/// Cap on the commit list, well inside GitHub's 65536-character body limit.
10864const COMMITS_MAX_CHARS: usize = 20_000;
10865
10866/// Read the commits and diff stat of `base..branch`. `None` when git cannot
10867/// say or finds nothing, so the caller falls back to what the run recorded.
10868async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10869    let commits = git::commit_log(repo, base, branch).await.ok()?;
10870    if commits.is_empty() {
10871        return None;
10872    }
10873    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10874    let lines: Vec<&str> = stat.lines().collect();
10875    let stat = if lines.len() > STAT_FILE_LINES + 1 {
10876        let omitted = lines.len() - 1 - STAT_FILE_LINES;
10877        let more = format!(" ... {omitted} more file(s)");
10878        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10879        kept.push(&more);
10880        kept.push(lines[lines.len() - 1]);
10881        kept.join("\n")
10882    } else {
10883        lines.join("\n")
10884    };
10885    Some(BranchFacts { commits, stat })
10886}
10887
10888/// Defang what would break the surrounding markdown: a closing `</details>`
10889/// and a code fence.
10890fn markdown_safe(text: &str) -> String {
10891    text.replace("</details>", "&lt;/details&gt;")
10892        .replace("\x60\x60\x60", "~~~")
10893}
10894
10895fn neutral_title(state: &RunState, winner: char) -> String {
10896    format!(
10897        "chore: land candidate {} of run {}",
10898        winner.to_ascii_uppercase(),
10899        state.id
10900    )
10901}
10902
10903/// Reuse the graph's bounded fixer without starting another graph or driver.
10904/// Copy the working state back even on errors so spent rounds remain visible.
10905pub(crate) async fn repair_landing_gate(
10906    state: &mut RunState,
10907    winner: &Candidate,
10908    outcomes: &[CommandOutcome],
10909) -> Result<GateFix> {
10910    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
10911        return Ok(GateFix::Stop);
10912    }
10913    let mut runner = Runner {
10914        roles: state.config.resolve_roles()?,
10915        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
10916        state: state.clone(),
10917        pause: Pause::new(),
10918        interrupt: Pause::new(),
10919    };
10920    let result = runner.gate_fix_round(winner, outcomes).await;
10921    *state = runner.state;
10922    result
10923}
10924
10925/// Recheck a repair's e2e requirement across failures, resource deferrals and
10926/// process interruptions. A successful result only applies to its exact SHA.
10927pub(crate) async fn check_gate_repair_e2e(
10928    state: &mut RunState,
10929    winner: &Candidate,
10930    wait: &crate::repo_lock::WaitPolicy,
10931) -> Result<GateFix> {
10932    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
10933        return Ok(GateFix::Retry);
10934    }
10935    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10936    if state
10937        .gate_repair_e2e
10938        .as_ref()
10939        .is_some_and(|r| r.head == head && r.passed)
10940    {
10941        return Ok(GateFix::Retry);
10942    }
10943    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10944        head: head.clone(),
10945        passed: false,
10946        outcomes: Vec::new(),
10947    });
10948    state.save()?;
10949    let commands = state.config.verify.e2e.clone();
10950    let outcomes = if commands.is_empty() {
10951        Vec::new()
10952    } else {
10953        let shell = state.config.shell();
10954        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10955        let cache_dir = state.config.cache_dir();
10956        let (outcomes, _) = with_cache_lease(
10957            state,
10958            cache_dir.as_deref(),
10959            "e2e",
10960            "e2e",
10961            &winner.worktree,
10962            &head,
10963            timeout,
10964            "gate repair e2e",
10965            wait,
10966            |state, budget| async move {
10967                run_e2e_with_retry(
10968                    state,
10969                    &shell,
10970                    &commands,
10971                    &winner.worktree,
10972                    budget,
10973                    "gate repair e2e",
10974                )
10975                .await
10976            },
10977        )
10978        .await?;
10979        outcomes
10980    };
10981    let deferred = verify_inconclusive(&outcomes);
10982    let passed = outcomes.iter().all(CommandOutcome::ok)
10983        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
10984    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10985        head,
10986        passed,
10987        outcomes,
10988    });
10989    state.save()?;
10990    if deferred {
10991        Ok(GateFix::Defer)
10992    } else if !passed {
10993        state.event(
10994            "gate",
10995            "verify.e2e failed after gate repair; nothing published",
10996        );
10997        Ok(GateFix::Stop)
10998    } else {
10999        Ok(GateFix::Retry)
11000    }
11001}
11002
11003/// Run the same configured gate for a landing head and a fresh graph.
11004pub(crate) async fn run_landing_gate(
11005    state: &mut RunState,
11006    winner: &Candidate,
11007    wait: &crate::repo_lock::WaitPolicy,
11008) -> Result<Vec<CommandOutcome>> {
11009    let shell = state.config.shell();
11010    let gate_commands = state.config.verify.gate.clone();
11011    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11012    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11013        Vec::new()
11014    } else {
11015        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11016        let cache_dir = state.config.cache_dir();
11017        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11018        let (outcomes, _) = with_cache_lease(
11019            state,
11020            cache_dir.as_deref(),
11021            "gate",
11022            "gate",
11023            &winner.worktree,
11024            &head,
11025            timeout,
11026            "final gate",
11027            wait,
11028            |state, budget| {
11029                let shell = shell.clone();
11030                let gate_commands = gate_commands.clone();
11031                let winner = winner.clone();
11032                async move {
11033                    Runner::run_pre_gate(state, &winner).await;
11034                    let (outcomes, timed_out_pids) = run_commands(
11035                        state,
11036                        "gate",
11037                        "gate",
11038                        0,
11039                        &shell,
11040                        &gate_commands,
11041                        &winner.worktree,
11042                        budget,
11043                    )
11044                    .await;
11045                    (outcomes, false, timed_out_pids)
11046                }
11047            },
11048        )
11049        .await?;
11050        outcomes
11051    };
11052    if outcomes.is_empty() {
11053        // Nothing configured to check — distinct from every other
11054        // silence in this run's event log, since an empty `gate` alone
11055        // no longer says whether the gate ran at all (see
11056        // `RunState::gate_ran`'s own doc).
11057        state.event("gate", "no gate commands configured; not tested");
11058    }
11059    for o in &outcomes {
11060        state.event(
11061            "gate",
11062            format!(
11063                "`{}` -> {}",
11064                o.command,
11065                if o.ok() {
11066                    "pass".to_owned()
11067                } else {
11068                    format!(
11069                        "FAIL ({:?})\n{}",
11070                        o.code,
11071                        tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11072                    )
11073                }
11074            ),
11075        );
11076    }
11077    Ok(outcomes)
11078}
11079
11080/// The pull request title to hand to `land::merge_subject`. A review-only run
11081/// opened by an earlier build titled its pull request with the review prompt;
11082/// that title is dropped (empty, so the fallback applies) rather than landed.
11083/// Any other title, including an operator's rename, passes through untouched,
11084/// and so does every title of a run that implements a task.
11085pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11086    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11087        ""
11088    } else {
11089        pr_title
11090    }
11091}
11092
11093/// What the squash subject falls back to when the pull request title is empty
11094/// or candidate-shaped: for a review-only run the derived title, never the
11095/// review prompt held in `instruction`.
11096pub fn landing_subject_source(state: &RunState) -> String {
11097    if is_review_run(state) {
11098        let winner = state.candidates.first().map_or('A', |c| c.label);
11099        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11100    }
11101    state.instruction.clone()
11102}
11103
11104/// `summary` without its `TITLE:` line, which the pull request title already
11105/// carries.
11106fn summary_without_title(summary: &str) -> String {
11107    let mut lines = summary.trim().lines().peekable();
11108    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11109        lines.next();
11110    }
11111    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11112}
11113
11114/// The pull request title and body for the winning candidate.
11115///
11116/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
11117/// the task's own opening line via [`queue::title_from`] when there is none.
11118/// `state.instruction` can open with blank lines (`task_text` only rejects a
11119/// body that is blank *entirely*), which `title_from` skips.
11120///
11121/// Publishing uses an independent description generated from the final diff
11122/// and structured evidence. Internal summaries and reports remain local.
11123/// Non-publishing previews may display the original task and candidate summary.
11124/// The footer retains the run and candidate provenance.
11125#[cfg(test)]
11126fn pr_message(state: &RunState, winner: char) -> PrMessage {
11127    pr_message_with(state, winner, None)
11128}
11129
11130/// [`pr_message`] with what the branch of a review-only run says about itself.
11131/// `facts` is ignored for a run that implements a task.
11132#[cfg(test)]
11133fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11134    let raw = pr_message_raw(
11135        state,
11136        winner,
11137        facts,
11138        crate::pr_outcome::FinalHead::Unreported,
11139        ChangeNote::NotAsked,
11140    );
11141    let id = crate::scrub::Identity::current();
11142    PrMessage {
11143        title: crate::scrub::scrub(&raw.title, &id),
11144        body: crate::scrub::scrub(&raw.body, &id),
11145    }
11146}
11147
11148/// What the public description may say about the change itself.
11149#[derive(Debug, Clone, Copy)]
11150enum ChangeNote<'a> {
11151    /// A local preview or non-publishing merge mode: no public generation needed.
11152    NotAsked,
11153    /// A summary written from the final diff.
11154    Summary(&'a crate::run::ChangeSummary),
11155    /// A summary was asked for and could not be produced; the text says why.
11156    Unavailable(&'a str),
11157}
11158
11159fn pr_message_raw(
11160    state: &RunState,
11161    winner: char,
11162    facts: Option<&BranchFacts>,
11163    head: crate::pr_outcome::FinalHead<'_>,
11164    change: ChangeNote<'_>,
11165) -> PrMessage {
11166    if !matches!(change, ChangeNote::NotAsked) {
11167        let (title, mut body) = match change {
11168            ChangeNote::Summary(c) => (
11169                c.title.clone(),
11170                format!("## Summary\n\n{}\n\n", c.body.trim()),
11171            ),
11172            ChangeNote::Unavailable(why) => (
11173                neutral_title(state, winner),
11174                format!(
11175                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11176                ),
11177            ),
11178            ChangeNote::NotAsked => unreachable!(),
11179        };
11180        body.push_str(&public_pr_evidence(state, head));
11181        body.push_str(&format!(
11182            "\n---\nmagi:run/{} magi:candidate-{}\n",
11183            state.id,
11184            winner.to_ascii_lowercase()
11185        ));
11186        return PrMessage { title, body };
11187    }
11188    pr_preview(state, winner, facts, head)
11189}
11190
11191/// Append public facts from the run records after every prose generation,
11192/// including posting-gate rewrites. The agent cannot drop this provenance.
11193fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11194    let mut body = crate::pr_outcome::review_rounds(state);
11195    body.push_str(&crate::pr_outcome::final_verification(state, head));
11196    let open = state.last_round_findings();
11197    if !open.is_empty() {
11198        body.push_str("## Open review findings\n\n");
11199        for f in open {
11200            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11201        }
11202        body.push('\n');
11203    }
11204    if let Some(c) = state.change_summary.as_ref()
11205        && head == crate::pr_outcome::FinalHead::At(&c.head)
11206    {
11207        body.push_str(&format!(
11208            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11209            c.base, c.head
11210        ));
11211    }
11212    body
11213}
11214
11215/// Local-only preview of the operator's task and candidate summary.
11216fn pr_preview(
11217    state: &RunState,
11218    winner: char,
11219    facts: Option<&BranchFacts>,
11220    head: crate::pr_outcome::FinalHead<'_>,
11221) -> PrMessage {
11222    let summary = state
11223        .candidates
11224        .iter()
11225        .find(|c| c.label == winner)
11226        .map(|c| c.summary.as_str())
11227        .unwrap_or_default();
11228    // The fallback is the operator's own words and may not be English; GitHub
11229    // text always is, so a non-English task gets a neutral title instead.
11230    let review = is_review_run(state);
11231    let title = if review {
11232        facts
11233            .and_then(|f| {
11234                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11235            })
11236            .or_else(|| review_title(state))
11237            .or_else(|| {
11238                state
11239                    .candidates
11240                    .iter()
11241                    .find(|c| c.label == winner)
11242                    .filter(|c| !c.branch.starts_with("magi/"))
11243                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11244            })
11245            .unwrap_or_else(|| neutral_title(state, winner))
11246    } else {
11247        summary_title(summary).unwrap_or_else(|| {
11248            english_title(
11249                &state.instruction,
11250                PR_TITLE_MAX,
11251                state.config.graph.github_english(),
11252            )
11253            .unwrap_or_else(|| neutral_title(state, winner))
11254        })
11255    };
11256
11257    let mut body = String::new();
11258    let what = summary_without_title(summary);
11259    if !what.is_empty() {
11260        body.push_str("## Summary\n\n");
11261        body.push_str(&what);
11262        body.push_str("\n\n");
11263    }
11264
11265    // Review history uses structured facts across all rounds, never reports.
11266    body.push_str(&crate::pr_outcome::review_rounds(state));
11267    body.push_str(&crate::pr_outcome::final_verification(state, head));
11268    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11269
11270    let open = state.open_findings();
11271    if !open.is_empty() {
11272        body.push_str("## Open review findings\n\n");
11273        for f in &open {
11274            body.push_str(&format!(
11275                "- `{}` [{:?}] {}\n",
11276                f.id, f.severity, "Context retained in local review records."
11277            ));
11278        }
11279        body.push('\n');
11280    }
11281
11282    if let Some(fix) = fix
11283        && !fix.rejected.is_empty()
11284    {
11285        body.push_str("## Declined by the fixer\n\n");
11286        for r in &fix.rejected {
11287            body.push_str(&format!(
11288                "- `{}`: {}\n",
11289                r.id, "Reason retained in local review records."
11290            ));
11291        }
11292        body.push('\n');
11293    }
11294
11295    if review {
11296        // The review prompt is not the task; list what the branch carries.
11297        body.push_str("## Commits under review\n\n");
11298        if let Some(facts) = facts {
11299            let mut left = COMMITS_MAX_CHARS;
11300            for (i, (subject, text)) in facts.commits.iter().enumerate() {
11301                let mut entry = format!("- {}\n", markdown_safe(subject));
11302                for l in markdown_safe(text).lines() {
11303                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
11304                }
11305                if left == 0 {
11306                    body.push_str(&format!(
11307                        "- ... {} more commit(s)\n",
11308                        facts.commits.len() - i
11309                    ));
11310                    break;
11311                }
11312                if entry.len() > left {
11313                    // Even the first commit is cut: one huge body must not
11314                    // push the whole description past GitHub's limit.
11315                    let mut end = left;
11316                    while !entry.is_char_boundary(end) {
11317                        end -= 1;
11318                    }
11319                    entry.truncate(end);
11320                    entry.push_str("\n  ... (truncated)\n");
11321                    left = 0;
11322                } else {
11323                    left -= entry.len();
11324                }
11325                body.push_str(&entry);
11326            }
11327            if !facts.stat.trim().is_empty() {
11328                body.push_str(&format!(
11329                    "\n## Diff stat\n\n```\n{}\n```\n",
11330                    markdown_safe(facts.stat.trim())
11331                ));
11332            }
11333        } else {
11334            match &state.reviewed_commits {
11335                Some(subjects) => {
11336                    for s in subjects {
11337                        body.push_str(&format!("- {}\n", s.trim()));
11338                    }
11339                }
11340                None => {
11341                    // An older run kept only the prompt, with the commit list
11342                    // after its first paragraph.
11343                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11344                    body.push_str(rest.trim());
11345                    body.push('\n');
11346                }
11347            }
11348        }
11349    } else {
11350        let task = state.instruction.trim();
11351        let task = if task.is_empty() {
11352            "(empty task)"
11353        } else {
11354            task
11355        };
11356        body.push_str(&format!(
11357            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11358            task.replace("</details>", "&lt;/details&gt;")
11359        ));
11360    }
11361
11362    body.push_str(&format!(
11363        "\n---\nmagi:run/{} magi:candidate-{}\n",
11364        state.id,
11365        winner.to_ascii_lowercase()
11366    ));
11367
11368    PrMessage { title, body }
11369}
11370
11371/// Source imports have already been prepared from structured inputs. Legacy
11372/// inferred references must never instruct a seat to reconstruct those imports.
11373fn seeded_instruction(state: &RunState) -> String {
11374    let mut inputs = Vec::new();
11375    if let Some(start) = &state.sources.start_from {
11376        inputs.push(format!(
11377            "Start from `{}` at `{}` (including ancestry).",
11378            start.token, start.sha
11379        ));
11380    }
11381    for pick in &state.sources.pick_commits {
11382        inputs.push(format!(
11383            "Pick `{}` at `{}` (single commit).",
11384            pick.token, pick.sha
11385        ));
11386    }
11387    if inputs.is_empty() {
11388        state.instruction.clone()
11389    } else {
11390        format!(
11391            "{}\n\n# Explicit source inputs already prepared\n\n{}",
11392            state.instruction,
11393            inputs.join("\n")
11394        )
11395    }
11396}
11397
11398/// Does the winner have no commits ahead of the base it would land on?
11399/// Any failure to find out reads as "not empty": the merge then behaves as it
11400/// always did rather than refusing on a guess.
11401async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11402    let base = &state.base_branch;
11403    let mut against = base.clone();
11404    if mode == MergeMode::Pr {
11405        // A pull request lands on the remote's base, never the local branch
11406        // of the same name: if that cannot be read, "not empty" is the safe
11407        // answer.
11408        let remote = &state.config.merge.remote;
11409        let tracking = format!("{remote}/{base}");
11410        let fetched = git::fetch(repo, remote, base).await;
11411        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11412            against = tracking;
11413        } else {
11414            return false;
11415        }
11416    }
11417    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11418}
11419
11420/// Why nothing was opened for an empty winner, with what the task's own
11421/// references resolved to.
11422fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11423    let mut detail = format!(
11424        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11425         nothing to open a pull request for"
11426    );
11427    detail.push_str(
11428        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11429    );
11430    detail
11431}
11432
11433/// What the `Pr` merge does once it knows whether the branch already has an
11434/// open pull request.
11435#[derive(Debug, PartialEq, Eq)]
11436enum PrPlan {
11437    Create,
11438    Adopt { url: String, title: String },
11439    Stop(String),
11440}
11441
11442/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
11443/// a failed lookup -> stop with the real reason. Never guesses.
11444fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11445    match found {
11446        Ok(land::OpenPr::None) => PrPlan::Create,
11447        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11448        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11449            "several open pull requests exist for this branch, not picking one: {}",
11450            urls.join(" ")
11451        )),
11452        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11453    }
11454}
11455
11456/// `gh pr create`, returning the PR url.
11457pub(crate) async fn gh_pr_create(
11458    cwd: &Path,
11459    base: &str,
11460    head: &str,
11461    title: &str,
11462    body: &str,
11463) -> Result<String> {
11464    let out = tokio::process::Command::new("gh")
11465        .args([
11466            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11467        ])
11468        .current_dir(cwd)
11469        .quiet()
11470        .stdin(std::process::Stdio::null())
11471        .output()
11472        .await
11473        .context("spawn gh")?;
11474    if out.status.success() {
11475        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11476    } else {
11477        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11478    }
11479}
11480
11481/// Never force deletion of uncommitted work, including unknown legacy trees.
11482async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11483    match std::fs::symlink_metadata(path) {
11484        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11485            // A previous cleanup may have removed the tree before a branch
11486            // deletion or state save failed. Unregister it if still present.
11487            git::worktree_remove_clean(repo, path).await?;
11488            return Ok(());
11489        }
11490        Err(e) => return Err(e.into()),
11491        Ok(_) => {}
11492    }
11493    if !git::is_clean(path).await? {
11494        bail!(
11495            "worktree has uncommitted work; refusing removal: {}",
11496            path.display()
11497        );
11498    }
11499    if !git::worktree_remove_clean(repo, path).await? {
11500        bail!("worktree removal refused: {}", path.display());
11501    }
11502    Ok(())
11503}
11504
11505/// Tear a run's worktrees and branches down.
11506///
11507/// `home` is where the updated `run.json` is saved (via
11508/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
11509/// a housekeeping pass already has its own honest `home` handed to it, and
11510/// falling through to the global here would write back through whichever
11511/// directory some other process or test pinned into that `OnceLock` first,
11512/// not the one the caller actually resolved its `runs` and `state` from.
11513pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11514    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11515        bail!("refusing fold: {reason}");
11516    }
11517    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11518        bail!("candidate preservation incomplete; refusing cleanup");
11519    }
11520    // Serialize ordinary folding with the same starts/resumes as target-only
11521    // eviction. Otherwise a second janitor could remove the whole worktree
11522    // around a live output borrower even though target eviction was fenced.
11523    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11524    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11525    let _driver = crate::driver::maintenance(home, &state.id)?;
11526    let mut output_guards = Vec::new();
11527    for entry in std::fs::read_dir(state.worktree_root())
11528        .into_iter()
11529        .flatten()
11530        .flatten()
11531    {
11532        if entry.path().join(".git").is_file() {
11533            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11534        }
11535    }
11536    let repo = state.repo.clone();
11537    // CLI, Web and automatic cleanup all enter here, independently of
11538    // Runner::fold_losers. Protect deletion and shared config release too.
11539    let wait = repository_wait(state, None);
11540    let _coordination = match coordinate_operation(state, &repo, &wait).await {
11541        Ok(lease) => lease,
11542        Err(e) => {
11543            state.save_under(home)?;
11544            return Err(e);
11545        }
11546    };
11547    // Confirmed merges dispose of their owned sources after durable evidence
11548    // capture. Preflight every source before removing anything.
11549    let _source_outputs = if state.status == RunStatus::Merged
11550        && state.merge.as_ref().is_some_and(|merge| merge.ok)
11551    {
11552        verify_merged_fold_sources(state, drop_winner, home).await?
11553    } else {
11554        Vec::new()
11555    };
11556    let root = state.worktree_root();
11557    let winner = state.tally.as_ref().map(|t| t.winner);
11558    let mut removed = Vec::new();
11559
11560    for i in 0..state.candidates.len() {
11561        let c = state.candidates[i].clone();
11562        let is_winner = Some(c.label) == winner;
11563        if is_winner && !drop_winner {
11564            continue;
11565        }
11566        let existed = c.worktree.exists();
11567        // A vanished directory can still have a Git registration that keeps
11568        // its branch checked out. Remove that registration before the branch.
11569        remove_preserved_worktree(&repo, &c.worktree).await?;
11570        if existed {
11571            removed.push(c.worktree.to_string_lossy().into_owned());
11572        }
11573        // A branch handed to a later run (and its pull request) is not this
11574        // run's to delete.
11575        let handed_over = state.released_branches.contains(&c.branch);
11576        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11577            if !git::branch_delete(&repo, &c.branch).await? {
11578                bail!("candidate branch removal refused");
11579            }
11580            removed.push(c.branch.clone());
11581        }
11582        state.candidates[i].folded = true;
11583    }
11584
11585    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11586        let path = name.path();
11587        let keep = !drop_winner
11588            && winner.is_some_and(|w| {
11589                path.file_name()
11590                    .is_some_and(|n| n == format!("cand-{w}").as_str())
11591            });
11592        if keep {
11593            continue;
11594        }
11595        remove_preserved_worktree(&repo, &path).await?;
11596        removed.push(path.to_string_lossy().into_owned());
11597    }
11598
11599    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
11600    // judge worktrees, so once the loop above has cleared all of them out,
11601    // the parent is a bare directory nobody else was ever going to remove -
11602    // git only ever managed what was inside it. Left alone, one of these
11603    // accumulates per fully-folded run; the operator's own machine had 74.
11604    // `remove_if_empty` re-checks rather than assuming: a run whose winner
11605    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
11606    // so does anything a run never claimed that happens to share the bay.
11607    remove_if_empty(&root);
11608
11609    if state.enabled_worktree_config && drop_winner {
11610        // A release, not a raw disable: some sibling run in this repository
11611        // may still hold its own reference (see `git::acquire_worktree_config`),
11612        // and only the last release actually turns the setting back off.
11613        git::release_worktree_config(&repo).await.ok();
11614        state.enabled_worktree_config = false;
11615    }
11616    state.save_under(home)?;
11617    Ok(removed)
11618}
11619
11620async fn verify_merged_fold_sources(
11621    state: &RunState,
11622    drop_winner: bool,
11623    home: &Path,
11624) -> Result<Vec<std::fs::File>> {
11625    let winner = state.tally.as_ref().map(|t| t.winner);
11626    let mut heads = BTreeSet::new();
11627    for candidate in &state.candidates {
11628        if Some(candidate.label) == winner && !drop_winner {
11629            continue;
11630        }
11631        anyhow::ensure!(
11632            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11633            "candidate recovery source is outside the inventoried run root: {}",
11634            candidate.worktree.display()
11635        );
11636        match std::fs::symlink_metadata(&candidate.worktree) {
11637            Ok(_) => {}
11638            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11639            Err(error) => return Err(error.into()),
11640        }
11641        if !state.released_branches.contains(&candidate.branch)
11642            && git::branch_exists(&state.repo, &candidate.branch).await?
11643        {
11644            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11645        }
11646    }
11647    let mut cargo_guards = Vec::new();
11648    let entries = match std::fs::read_dir(state.worktree_root()) {
11649        Ok(entries) => Some(entries),
11650        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11651        Err(error) => return Err(error.into()),
11652    };
11653    for entry in entries.into_iter().flatten() {
11654        let path = entry?.path();
11655        let metadata = std::fs::symlink_metadata(&path)?;
11656        anyhow::ensure!(
11657            metadata.is_dir() && !metadata.file_type().is_symlink(),
11658            "non-directory or symlink recovery source retained: {}",
11659            path.display()
11660        );
11661        if !drop_winner
11662            && winner.is_some_and(|w| {
11663                path.file_name()
11664                    .is_some_and(|name| name == format!("cand-{w}").as_str())
11665            })
11666        {
11667            continue;
11668        }
11669        anyhow::ensure!(
11670            path.join(".git").is_file(),
11671            "unregistered recovery source retained: {}",
11672            path.display()
11673        );
11674        anyhow::ensure!(
11675            git::git(
11676                &path,
11677                &[
11678                    "status",
11679                    "--porcelain",
11680                    "--untracked-files=all",
11681                    "--ignored",
11682                    "--",
11683                    ".",
11684                    ":(exclude)target"
11685                ]
11686            )
11687            .await?
11688            .is_empty(),
11689            "source changes or ignored recovery files retained: {}",
11690            path.display()
11691        );
11692        match std::fs::symlink_metadata(path.join("target")) {
11693            Ok(_) => {
11694                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11695            }
11696            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11697            Err(error) => return Err(error.into()),
11698        }
11699        for checkpoint in [
11700            "rebase-merge",
11701            "rebase-apply",
11702            "sequencer",
11703            "CHERRY_PICK_HEAD",
11704            "MERGE_HEAD",
11705            "REBASE_HEAD",
11706            "REVERT_HEAD",
11707            "BISECT_LOG",
11708            "AUTO_MERGE",
11709            "MERGE_MSG",
11710        ] {
11711            let location = git::git(
11712                &path,
11713                &[
11714                    "rev-parse",
11715                    "--path-format=absolute",
11716                    "--git-path",
11717                    checkpoint,
11718                ],
11719            )
11720            .await?;
11721            anyhow::ensure!(
11722                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11723                if e.kind() == std::io::ErrorKind::NotFound),
11724                "Git recovery checkpoint retained: {checkpoint}"
11725            );
11726        }
11727        heads.insert(git::rev_parse(&path, "HEAD").await?);
11728    }
11729    crate::fold_evidence::preserve(state, home, &heads).await?;
11730
11731    Ok(cargo_guards)
11732}
11733
11734/// Remove `dir` if it exists and has nothing in it.
11735///
11736/// Best-effort and silent by design: a directory that is not empty (a run
11737/// whose winner is still parked there, a stray file some other process left)
11738/// is exactly the case this must refuse, and a directory that is already gone
11739/// is not a failure worth reporting either. `std::fs::remove_dir` itself
11740/// already refuses a non-empty directory, so the emptiness check below is
11741/// belt, not suspenders - it is what keeps this from ever attempting the
11742/// removal in the case that matters, rather than trusting `remove_dir`'s
11743/// error path to have no side effects if it ever changed.
11744fn remove_if_empty(dir: &Path) {
11745    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11746        std::fs::remove_dir(dir).ok();
11747    }
11748}
11749
11750/// Severity of the worst open finding in the last review round, for reporting.
11751pub fn worst_open(state: &RunState) -> Option<Severity> {
11752    state
11753        .reviews
11754        .last()?
11755        .reviews
11756        .iter()
11757        .flat_map(|r| r.findings.iter())
11758        .map(|f| f.severity)
11759        .max()
11760}
11761
11762/// `[worktree] setup` hides its products per worktree, which needs
11763/// `extensions.worktreeConfig`. Taken once per run and released with the run's
11764/// worktrees, like the hook's; every place that runs setup calls this first.
11765async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11766    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11767        git::acquire_worktree_config(repo).await?;
11768        state.enabled_worktree_config = true;
11769    }
11770    Ok(())
11771}
11772
11773#[cfg(test)]
11774mod tests {
11775    #[tokio::test]
11776    async fn launcher_base_fetch_requires_remote_success() {
11777        let tmp = tempfile::tempdir().unwrap();
11778        let repo = tmp.path().join("repo");
11779        let remote = tmp.path().join("remote");
11780        std::fs::create_dir(&repo).unwrap();
11781        std::fs::create_dir(&remote).unwrap();
11782        git::git(&remote, &["init", "--bare", "-b", "main"])
11783            .await
11784            .unwrap();
11785        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11786        git::git(
11787            &repo,
11788            &[
11789                "-c",
11790                "user.name=Fixture",
11791                "-c",
11792                "user.email=fixture@example.invalid",
11793                "commit",
11794                "--allow-empty",
11795                "-m",
11796                "fixture",
11797            ],
11798        )
11799        .await
11800        .unwrap();
11801        git::git(
11802            &repo,
11803            &["remote", "add", "origin", remote.to_str().unwrap()],
11804        )
11805        .await
11806        .unwrap();
11807        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11808        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11809        assert_eq!(
11810            resolve_base(&repo, "main", "origin", None)
11811                .await
11812                .unwrap()
11813                .commit,
11814            head
11815        );
11816        #[cfg(unix)]
11817        {
11818            let policy: crate::config::Coordination = toml::from_str(
11819                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11820            ).unwrap();
11821            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11822            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11823            let held = crate::repo_lock::SeatLease::operation_with(
11824                &repo,
11825                Some(&policy),
11826                &crate::repo_lock::WaitPolicy::default(),
11827            )
11828            .await
11829            .unwrap();
11830            assert!(held.is_some());
11831            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11832            tokio::pin!(fetch);
11833            assert!(
11834                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11835                    .await
11836                    .is_err()
11837            );
11838            assert!(
11839                !repo.join(".git/FETCH_HEAD").exists(),
11840                "no fetch before admission"
11841            );
11842            drop(held);
11843            let fetched = fetch.await.unwrap();
11844            assert_eq!(fetched.commit, head);
11845            assert_eq!(fetched.remote, "origin");
11846            assert_eq!(fetched.branch, "main");
11847        }
11848        // A surviving tracking ref is not successful fetch evidence.
11849        git::git(
11850            &repo,
11851            &[
11852                "remote",
11853                "set-url",
11854                "origin",
11855                tmp.path().join("missing").to_str().unwrap(),
11856            ],
11857        )
11858        .await
11859        .unwrap();
11860        let error = resolve_base(&repo, "main", "origin", None)
11861            .await
11862            .unwrap_err()
11863            .to_string();
11864        assert!(error.contains("cannot read origin/main"));
11865        assert!(error.contains("refusing to branch off"));
11866    }
11867
11868    #[test]
11869    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11870        let leaked = vec!["chore(deps): update a crate".to_owned()];
11871        let own = "fix(daemon): apply a chosen action";
11872        assert!(should_retitle("", own, &leaked));
11873        assert!(should_retitle(
11874            &format!("{REVIEW_PROMPT_OPENING} `x`"),
11875            own,
11876            &leaked
11877        ));
11878        assert!(should_retitle(
11879            "chore: land candidate A of run 1",
11880            own,
11881            &leaked
11882        ));
11883        assert!(should_retitle(
11884            "magi: candidate A (uncommitted work)",
11885            own,
11886            &leaked
11887        ));
11888        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11889        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11890        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11891    }
11892
11893    #[test]
11894    fn pr_merge_plan_creates_adopts_or_stops() {
11895        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11896        assert_eq!(
11897            pr_merge_plan(Ok(land::OpenPr::One {
11898                url: "u".into(),
11899                title: "t".into()
11900            })),
11901            PrPlan::Adopt {
11902                url: "u".into(),
11903                title: "t".into()
11904            }
11905        );
11906        let PrPlan::Stop(many) =
11907            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11908        else {
11909            panic!("many must stop");
11910        };
11911        assert!(many.contains('a') && many.contains('b'));
11912        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11913            panic!("a failed lookup must stop");
11914        };
11915        assert!(err.contains("bad token"));
11916    }
11917
11918    use super::*;
11919    use crate::run::GateStatus;
11920    use std::collections::BTreeMap;
11921    use std::time::Duration;
11922
11923    fn conductor() -> AgentSpec {
11924        AgentSpec {
11925            id: "conductor".to_owned(),
11926            kind: crate::config::AgentKind::Command,
11927            model: None,
11928            command: vec!["true".to_owned()],
11929            extra_args: Vec::new(),
11930            env: BTreeMap::new(),
11931            prompt_delivery: None,
11932            max_turn_secs: None,
11933        }
11934    }
11935
11936    fn spec(id: &str) -> AgentSpec {
11937        AgentSpec {
11938            id: id.to_owned(),
11939            kind: crate::config::AgentKind::Command,
11940            model: None,
11941            command: vec!["true".to_owned()],
11942            extra_args: Vec::new(),
11943            env: BTreeMap::new(),
11944            prompt_delivery: None,
11945            max_turn_secs: None,
11946        }
11947    }
11948
11949    fn ids(xs: &[&str]) -> BTreeSet<String> {
11950        xs.iter().map(|s| (*s).to_owned()).collect()
11951    }
11952
11953    #[test]
11954    fn pick_successor_skips_an_agent_another_seat_holds() {
11955        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
11956        let roster = [spec("a"), spec("b"), spec("c")];
11957        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11958        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11959    }
11960
11961    #[test]
11962    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11963        // The other seat started on c but was handed to d; c is free again.
11964        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11965        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11966        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11967        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11968        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11969    }
11970
11971    #[test]
11972    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11973        let roster = [spec("a"), spec("b")];
11974        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11975        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11976        let carried = ids(&["b"]);
11977        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11978        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11979    }
11980
11981    #[test]
11982    fn pick_successor_returns_none_without_any_untried_successor() {
11983        let roster = [spec("a"), spec("b")];
11984        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11985        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11986        let carried = ids(&["a"]);
11987        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11988    }
11989
11990    #[test]
11991    fn pick_successor_rescue_avoids_another_seats_occupant() {
11992        // b is a carried failure and free; a is held by the other seat.
11993        let roster = [spec("a"), spec("b"), spec("c")];
11994        let carried = ids(&["b", "c"]);
11995        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11996        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11997    }
11998
11999    #[test]
12000    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12001        let roster = [spec("a"), spec("b"), spec("c")];
12002        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12003        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12004    }
12005
12006    #[test]
12007    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12008        let roster = [spec("a"), spec("b"), spec("c")];
12009        let failed = ids(&["a", "b", "c"]);
12010        // Rescue looks at the whole roster, once per id, then runs out.
12011        let mut tried = ids(&["b"]);
12012        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12013        assert_eq!(first.id, "a");
12014        tried.insert(first.id.clone());
12015        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12016        assert_eq!(second.id, "c");
12017        tried.insert(second.id.clone());
12018        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12019    }
12020
12021    #[test]
12022    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12023        let roster = [spec("a"), spec("b")];
12024        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12025        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12026    }
12027
12028    #[test]
12029    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12030        let roster = [spec("a"), spec("b"), spec("c")];
12031        let h = SeatHistory {
12032            failed: ids(&["a"]),
12033            last_ok: Some("b".to_owned()),
12034            last_fail: None,
12035        };
12036        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12037        // A last answerer that left the roster, or later failed, is ignored.
12038        let gone = SeatHistory {
12039            last_ok: Some("zzz".to_owned()),
12040            ..h.clone()
12041        };
12042        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12043        let failed = SeatHistory {
12044            failed: ids(&["a", "b"]),
12045            last_ok: Some("b".to_owned()),
12046            last_fail: None,
12047        };
12048        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12049    }
12050
12051    #[test]
12052    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12053        let first = SeatState::new("review-1", "alpha", 7);
12054        let next = handover_seat("review-1", "gamma", 7);
12055        assert_ne!(first.claude_session, next.claude_session);
12056    }
12057
12058    #[test]
12059    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12060        let mut state = state_with_summary("x", "y");
12061        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12062        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12063        assert_ne!(a.claude_session, b.claude_session);
12064    }
12065
12066    #[test]
12067    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12068        let roster = [spec("a"), spec("b")];
12069        let h = SeatHistory {
12070            failed: ids(&["a", "b"]),
12071            ..SeatHistory::default()
12072        };
12073        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12074        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12075        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12076    }
12077
12078    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12079    // fallback loop depends on to terminate: it must walk forward from the
12080    // seat's own position, never restart at the front of the roster, and it
12081    // must never hand back an id already tried, however many times that id
12082    // happens to appear.
12083
12084    #[test]
12085    fn failure_signature_ignores_numbers_and_paths() {
12086        assert_eq!(
12087            failure_signature("exited with Some(2) and no usable output"),
12088            failure_signature("exited with Some(137) and no usable output")
12089        );
12090        assert_eq!(
12091            failure_signature("process exited with no usable output (exit code 2)"),
12092            failure_signature("process exited with no usable output (exit code 137)")
12093        );
12094        for (old, new) in [
12095            (
12096                "exited with Some(2) and no usable output",
12097                "process exited with no usable output (exit code 2)",
12098            ),
12099            (
12100                "exited with None and no usable output",
12101                "process exited with no usable output (exit code unavailable)",
12102            ),
12103        ] {
12104            let previous = FailClass::Other(failure_signature(old));
12105            let current = FailClass::Other(failure_signature(new));
12106            assert_eq!(previous, current, "resumption retains the failure class");
12107            assert!(!should_hand_over(Some(&previous), &current));
12108        }
12109        assert_eq!(
12110            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12111            failure_signature("cannot open /var/x.txt: denied")
12112        );
12113        assert_ne!(failure_signature("boom"), failure_signature("bang"));
12114    }
12115
12116    #[test]
12117    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12118        let other = FailClass::Other("x".into());
12119        assert!(should_hand_over(None, &FailClass::Quota));
12120        assert!(should_hand_over(Some(&other), &FailClass::Quota));
12121        assert!(should_hand_over(
12122            Some(&FailClass::Timeout),
12123            &FailClass::Timeout
12124        ));
12125        assert!(should_hand_over(None, &other));
12126        assert!(!should_hand_over(Some(&other), &other));
12127        assert!(should_hand_over(
12128            Some(&other),
12129            &FailClass::Other("y".into())
12130        ));
12131        // A quota or timeout in between ends the run of identical failures.
12132        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12133        assert!(should_hand_over(Some(&FailClass::Quota), &other));
12134    }
12135
12136    #[test]
12137    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12138        let a = SeatState::new("judge-1", "alpha", 7);
12139        let b = handover_seat("judge-1", "beta", 7);
12140        assert_ne!(a.claude_session, b.claude_session);
12141        assert_eq!(b.turns, 0);
12142    }
12143
12144    #[test]
12145    fn a_timeout_is_classified_apart_from_other_failures() {
12146        assert_eq!(
12147            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12148            Some(FailClass::Timeout)
12149        );
12150        assert!(matches!(
12151            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12152            Some(FailClass::Other(_))
12153        ));
12154    }
12155
12156    #[test]
12157    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12158        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12159        let tried = BTreeSet::from(["beta".to_owned()]);
12160        // beta sits at index 1; the next candidate is gamma, never alpha —
12161        // which is very likely a different candidate slot's own agent.
12162        let next = next_untried_in_roster(&roster, 1, &tried);
12163        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12164    }
12165
12166    #[test]
12167    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12168        let roster = vec![spec("alpha"), spec("beta")];
12169        let tried = BTreeSet::from(["beta".to_owned()]);
12170        // beta is the roster's last entry: nothing follows it, and alpha —
12171        // earlier in the roster, almost certainly a different candidate
12172        // slot's own agent — must not be reached by wrapping back to it.
12173        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12174    }
12175
12176    #[test]
12177    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12178        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12179        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12180        // beta (index 1) and gamma (index 2, the only entry after it) have
12181        // both been tried; alpha (index 0) never has, but it comes before
12182        // beta's own position, so there is nothing further for this seat.
12183        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12184    }
12185
12186    #[test]
12187    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12188        let roster = vec![spec("a"), spec("a"), spec("b")];
12189        let tried = BTreeSet::from(["a".to_owned()]);
12190        let next = next_untried_in_roster(&roster, 0, &tried);
12191        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12192    }
12193
12194    #[test]
12195    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12196        let roster = vec![spec("a"), spec("b")];
12197        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12198        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12199    }
12200
12201    #[test]
12202    fn remove_if_empty_only_ever_takes_a_bare_directory() {
12203        let dir = tempfile::tempdir().unwrap();
12204        let bay = dir.path().join("ffff");
12205
12206        // Not there yet: nothing to do, nothing to panic on.
12207        remove_if_empty(&bay);
12208        assert!(!bay.exists());
12209
12210        // Something still inside - the winner's worktree, or a stray file -
12211        // keeps the directory standing.
12212        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12213        remove_if_empty(&bay);
12214        assert!(bay.exists(), "non-empty directory must survive");
12215
12216        // Once the last entry is gone, so is the directory itself.
12217        std::fs::remove_dir(bay.join("cand-A")).unwrap();
12218        remove_if_empty(&bay);
12219        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12220    }
12221
12222    // `round_is_clean` is the exact decision this task fixed: a round with a
12223    // seat that never answered must not read the same as a round every seat
12224    // actually reviewed. These are deterministic and process-free by design —
12225    // the equivalent end-to-end check (a real reviewer timing out under a
12226    // live graph run) is a genuine race against wall-clock contention, and a
12227    // spawn slow enough to blow even a generous budget under a loaded test
12228    // run must not turn this specific regression check flaky.
12229
12230    #[test]
12231    fn a_full_panel_that_found_nothing_is_clean() {
12232        assert!(round_is_clean(
12233            0,
12234            true,
12235            2,
12236            2,
12237            0,
12238            IncompleteReviewPolicy::Block
12239        ));
12240    }
12241
12242    #[test]
12243    fn a_missing_seat_is_never_clean_under_the_default_policy() {
12244        assert!(!round_is_clean(
12245            0,
12246            true,
12247            1,
12248            2,
12249            0,
12250            IncompleteReviewPolicy::Block
12251        ));
12252    }
12253
12254    #[test]
12255    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12256        assert!(!round_is_clean(
12257            1,
12258            true,
12259            1,
12260            2,
12261            0,
12262            IncompleteReviewPolicy::Warn
12263        ));
12264    }
12265
12266    #[test]
12267    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12268        assert!(round_is_clean(
12269            0,
12270            true,
12271            1,
12272            2,
12273            0,
12274            IncompleteReviewPolicy::Warn
12275        ));
12276    }
12277
12278    #[test]
12279    fn a_full_panel_with_an_open_finding_is_not_clean() {
12280        assert!(!round_is_clean(
12281            1,
12282            true,
12283            2,
12284            2,
12285            0,
12286            IncompleteReviewPolicy::Block
12287        ));
12288    }
12289
12290    #[test]
12291    fn a_full_panel_with_a_red_e2e_is_not_clean() {
12292        assert!(!round_is_clean(
12293            0,
12294            false,
12295            2,
12296            2,
12297            0,
12298            IncompleteReviewPolicy::Block
12299        ));
12300    }
12301
12302    // The stall this task closes: under the default `block` policy, a seat
12303    // missing only because it was rate limited must not force a wait for a
12304    // session limit that will not lift by the next round. `round_is_clean`
12305    // is where that quorum carve-out lives; the review loop around it never
12306    // changes what a reviewer's vote or a finding's severity means.
12307
12308    #[test]
12309    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12310        // 1 of 2 answered, and the one missing was quota'd — the exact
12311        // "review-2 rate limited (quota)" shape from the field report.
12312        assert!(round_is_clean(
12313            0,
12314            true,
12315            1,
12316            2,
12317            1,
12318            IncompleteReviewPolicy::Block
12319        ));
12320    }
12321
12322    #[test]
12323    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12324        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
12325        // not a quota loss (`quota_missing` stays 0) — worth another try.
12326        assert!(!round_is_clean(
12327            0,
12328            true,
12329            1,
12330            2,
12331            0,
12332            IncompleteReviewPolicy::Block
12333        ));
12334    }
12335
12336    #[test]
12337    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12338        assert!(!round_is_clean(
12339            1,
12340            true,
12341            1,
12342            2,
12343            1,
12344            IncompleteReviewPolicy::Block
12345        ));
12346        assert!(!round_is_clean(
12347            0,
12348            false,
12349            1,
12350            2,
12351            1,
12352            IncompleteReviewPolicy::Block
12353        ));
12354    }
12355
12356    #[test]
12357    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12358        // Every seat quota'd, nobody answered: there is no panel to decide
12359        // on, so this must fall through to the existing block-and-retry
12360        // fallback rather than call an unreviewed patch clean.
12361        assert!(!round_is_clean(
12362            0,
12363            true,
12364            0,
12365            2,
12366            2,
12367            IncompleteReviewPolicy::Block
12368        ));
12369    }
12370
12371    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12372        CommandOutcome {
12373            command: "test".to_owned(),
12374            code,
12375            output_tail: String::new(),
12376            duration_ms: 0,
12377            resource_blocked,
12378        }
12379    }
12380
12381    #[test]
12382    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12383        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12384        assert!(
12385            !verify_inconclusive(&[outcome(Some(1), false)]),
12386            "an ordinary failure is still evidence about the patch"
12387        );
12388        assert!(verify_inconclusive(&[outcome(None, true)]));
12389        assert!(
12390            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12391            "one inconclusive outcome taints the whole batch"
12392        );
12393        assert!(!verify_inconclusive(&[]));
12394    }
12395
12396    #[tokio::test]
12397    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12398        // Alive for the first two checks, then dead - confirms the loop
12399        // actually re-polls rather than deciding once and sleeping out the
12400        // ceiling regardless.
12401        let calls = std::sync::atomic::AtomicUsize::new(0);
12402        let started = Instant::now();
12403        wait_for_pids_with(
12404            &[123],
12405            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12406            Duration::from_millis(5),
12407            Duration::from_secs(5),
12408        )
12409        .await;
12410        assert!(
12411            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12412            "must keep checking rather than deciding on the first answer"
12413        );
12414        assert!(
12415            started.elapsed() < Duration::from_secs(1),
12416            "must return the moment it is confirmed dead, not wait out the ceiling"
12417        );
12418    }
12419
12420    #[tokio::test]
12421    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12422        let started = Instant::now();
12423        wait_for_pids_with(
12424            &[123],
12425            |_| true, // never reports dead
12426            Duration::from_millis(5),
12427            Duration::from_millis(30),
12428        )
12429        .await;
12430        let elapsed = started.elapsed();
12431        assert!(
12432            elapsed >= Duration::from_millis(30),
12433            "must not give up before its own ceiling: {elapsed:?}"
12434        );
12435        assert!(
12436            elapsed < Duration::from_secs(1),
12437            "must not wait past its own ceiling either: {elapsed:?}"
12438        );
12439    }
12440
12441    #[tokio::test]
12442    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12443        let started = Instant::now();
12444        wait_for_pids_with(
12445            &[],
12446            |_| true,
12447            Duration::from_secs(5),
12448            Duration::from_secs(5),
12449        )
12450        .await;
12451        assert!(
12452            started.elapsed() < Duration::from_millis(200),
12453            "an empty pid list has nothing to confirm"
12454        );
12455    }
12456
12457    // `review_conclusion` is the exact decision the review hand-off task
12458    // fixed: a round budget spent (or a tree that stopped moving) must not
12459    // collapse into `Blocked` regardless of what verification actually
12460    // said. Deterministic and process-free for the same reason the
12461    // `round_is_clean` family above is.
12462    fn review_round(
12463        clean: bool,
12464        blocking: usize,
12465        answered: usize,
12466        expected: usize,
12467        progressed: bool,
12468        e2e_ok: bool,
12469    ) -> ReviewRound {
12470        ReviewRound {
12471            round: 1,
12472            head: "h".to_owned(),
12473            verified_head: None,
12474            verified_at: None,
12475            reviews: Vec::new(),
12476            e2e: vec![CommandOutcome {
12477                command: "test".to_owned(),
12478                code: Some(if e2e_ok { 0 } else { 1 }),
12479                output_tail: String::new(),
12480                duration_ms: 0,
12481                resource_blocked: false,
12482            }],
12483            verify_retried: false,
12484            e2e_deferred: false,
12485            e2e_defer_reason: None,
12486            fix: None,
12487            blocking,
12488            answered,
12489            expected,
12490            clean,
12491            progressed,
12492            vote_split: false,
12493            reconsideration: Vec::new(),
12494            verdict: None,
12495        }
12496    }
12497
12498    #[test]
12499    fn review_conclusion_is_none_when_nothing_has_run() {
12500        assert_eq!(review_conclusion(&[], 3), None);
12501    }
12502
12503    #[test]
12504    fn review_conclusion_is_none_while_rounds_remain() {
12505        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12506        assert_eq!(review_conclusion(&rounds, 3), None);
12507    }
12508
12509    #[test]
12510    fn review_conclusion_is_gating_once_a_round_is_clean() {
12511        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12512        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12513    }
12514
12515    #[test]
12516    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12517        let rounds = vec![
12518            review_round(false, 1, 2, 2, true, true),
12519            review_round(false, 1, 2, 2, true, true),
12520        ];
12521        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12522    }
12523
12524    #[test]
12525    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12526        let rounds = vec![
12527            review_round(false, 1, 2, 2, true, true),
12528            review_round(false, 1, 2, 2, true, false),
12529        ];
12530        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12531    }
12532
12533    #[test]
12534    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12535        // Magi never got a command to run against this round's own head — a
12536        // resource-blocked attempt, not a red one — so this must never
12537        // settle on `Blocked` the way a genuine e2e failure would. `None`
12538        // here is what tells `Runner::review_loop` to retry the check
12539        // itself rather than trust this cheap recomputation with a verdict
12540        // it cannot actually produce.
12541        let mut blocked = review_round(false, 1, 2, 2, true, false);
12542        blocked.e2e[0].resource_blocked = true;
12543        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12544        assert_eq!(review_conclusion(&rounds, 2), None);
12545    }
12546
12547    #[test]
12548    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12549        // Missing input, not a verified tree — never a hand-off candidate.
12550        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12551        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12552    }
12553
12554    #[test]
12555    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12556        let rounds = vec![
12557            review_round(false, 1, 2, 2, false, true),
12558            review_round(false, 1, 2, 2, false, true),
12559        ];
12560        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12561    }
12562
12563    fn secs(n: u64) -> Duration {
12564        Duration::from_secs(n)
12565    }
12566
12567    /// A throwaway repo with one commit on `main`, for tests that need `merge`
12568    /// to make real (and, if it runs at all, real*ly fail*) git calls.
12569    #[tokio::test]
12570    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12571        ask_test_home();
12572        let tmp = tempfile::tempdir().unwrap();
12573        let repo = tmp.path().join("repo");
12574        std::fs::create_dir_all(&repo).unwrap();
12575        init_repo(&repo);
12576        let g = |args: &[&str]| {
12577            let out = std::process::Command::new("git")
12578                .args(args)
12579                .current_dir(&repo)
12580                .quiet()
12581                .output()
12582                .unwrap();
12583            assert!(
12584                out.status.success(),
12585                "{}",
12586                String::from_utf8_lossy(&out.stderr)
12587            );
12588        };
12589        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12590        g(&["checkout", "-b", "historical"]);
12591        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12592        g(&["add", "."]);
12593        g(&["commit", "-m", "old alternative"]);
12594        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12595        g(&["checkout", "-b", "magi/task-selected", &base]);
12596        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12597        g(&["add", "."]);
12598        g(&["commit", "-m", "measurement infrastructure"]);
12599        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12600        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12601        g(&["add", "."]);
12602        g(&["commit", "-m", "dependent implementation"]);
12603        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12604        g(&["checkout", "main"]);
12605        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12606        g(&["add", "."]);
12607        g(&["commit", "-m", "unrelated base change"]);
12608        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12609        let origin = tmp.path().join("origin.git");
12610        g(&[
12611            "clone",
12612            "--bare",
12613            repo.to_str().unwrap(),
12614            origin.to_str().unwrap(),
12615        ]);
12616        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12617        g(&["fetch", "origin"]);
12618        let selection = crate::pr_recovery::Selection {
12619            url: "https://example.test/project/pull/1".into(),
12620            branch: "magi/task-selected".into(),
12621            head: head.clone(),
12622            source_head: None,
12623            source_run: None,
12624            source_proof: None,
12625            disposition: Some(crate::pr_recovery::Disposition::Review),
12626            instruction: "preserve measured behavior".into(),
12627            approved_at: jiff::Timestamp::now(),
12628        };
12629        let mut template = runner_at(RunStatus::Reviewing);
12630        template.state.repo = repo.clone();
12631        template.state.base_commit = current.clone();
12632        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12633        template.state.origin = Some(crate::run::Origin::queue("selected"));
12634        template.state.pr_head = Some(selection.branch.clone());
12635        template.state.pr_recovery = Some(selection.clone());
12636        let mut runner =
12637            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12638                .await
12639                .unwrap();
12640        let wt = runner.state.winner().unwrap().worktree.clone();
12641        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12642        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12643        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12644        assert!(runner.state.seeds.is_empty());
12645        assert_eq!(runner.state.instruction, "preserve measured behavior");
12646        runner.sync_to_base().await.unwrap();
12647        assert_eq!(
12648            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12649            "prerequisite"
12650        );
12651        assert_eq!(
12652            std::fs::read_to_string(wt.join("implementation")).unwrap(),
12653            "measured implementation"
12654        );
12655        assert_eq!(
12656            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12657            "current main change"
12658        );
12659        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12660        assert_eq!(
12661            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12662            head
12663        );
12664
12665        // A failed selected-source setup stays inspectable, never resumable as a new competition.
12666        let mut partial = runner_at(RunStatus::Reviewing);
12667        partial.state.repo = repo.clone();
12668        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12669        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12670            run: Some("exit 1".into()),
12671            ..Default::default()
12672        }];
12673        let selection = runner.state.pr_recovery.clone().unwrap();
12674        partial.state.pr_recovery = Some(selection.clone());
12675        partial.state.pr_head = Some(selection.branch.clone());
12676        partial.state.origin = Some(crate::run::Origin::queue("selected"));
12677        let id = partial.state.id.clone();
12678        let error = match Runner::open_selected(
12679            &repo,
12680            partial.state,
12681            partial.roles,
12682            1,
12683            runner.state.base_commit.clone(),
12684            selection,
12685        )
12686        .await
12687        {
12688            Ok(_) => panic!("setup must fail"),
12689            Err(e) => e,
12690        };
12691        assert_eq!(
12692            error
12693                .downcast_ref::<crate::pr_recovery::StartupFailed>()
12694                .unwrap()
12695                .run,
12696            id
12697        );
12698        let failed = RunState::load(&id).unwrap();
12699        assert_eq!(failed.status, RunStatus::Blocked);
12700        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12701        assert!(Runner::resume(&id).is_err());
12702        runner.state.status = RunStatus::Ready;
12703        runner.state.driver_exited = true;
12704        runner.state.save().unwrap();
12705        let mut retry = runner_at(RunStatus::Reviewing);
12706        retry.state.repo = repo.clone();
12707        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12708        let mut reselected = runner.state.pr_recovery.clone().unwrap();
12709        reselected.approved_at = jiff::Timestamp::now();
12710        retry.state.origin = Some(crate::run::Origin::queue("selected"));
12711        retry.state.pr_head = Some(reselected.branch.clone());
12712        retry.state.pr_recovery = Some(reselected.clone());
12713        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12714            .await
12715            .unwrap();
12716        let restarted = Runner::open_selected(
12717            &repo,
12718            retry.state,
12719            retry.roles,
12720            1,
12721            runner.state.base_commit.clone(),
12722            reselected,
12723        )
12724        .await
12725        .unwrap();
12726        assert_eq!(
12727            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12728                .await
12729                .unwrap(),
12730            head
12731        );
12732        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12733        assert_eq!(
12734            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12735            head
12736        );
12737    }
12738
12739    #[tokio::test]
12740    #[cfg(unix)]
12741    async fn fallback_and_interrupted_restoration_require_coordination() {
12742        use std::os::fd::AsRawFd as _;
12743        ask_test_home();
12744        let tmp = tempfile::tempdir().unwrap();
12745        let mut runner = registered_runner(tmp.path()).await;
12746        runner.state.config.graph.timeout_resource_wait = 1;
12747        let winner = runner.state.candidates[0].clone();
12748        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12749        let leftover = winner.worktree.join("leftover");
12750        std::fs::write(&leftover, "untrusted edits").unwrap();
12751        let next = tmp.path().join("next-fixer");
12752        let mut failed = spec("first");
12753        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12754        let mut successor = spec("second");
12755        successor.command = vec![
12756            "sh".into(),
12757            "-c".into(),
12758            format!("touch '{}'; echo done", next.display()),
12759        ];
12760        let attempts = vec![
12761            (failed.clone(), "first".into()),
12762            (successor.clone(), "second".into()),
12763        ];
12764        let lock_path = runner.state.repo.join(".git/updater.lock");
12765        let external = std::fs::File::create(&lock_path).unwrap();
12766        assert_eq!(
12767            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12768            0
12769        );
12770        let artifacts = tmp.path().join("artifacts");
12771        let build = |spec, seat| SeatJob {
12772            spec,
12773            seat,
12774            cwd: winner.worktree.clone(),
12775            prompt: "fixture".into(),
12776            timeout: Duration::from_secs(10),
12777            allow_write: true,
12778            sessions: false,
12779            artifacts: artifacts.clone(),
12780            stem: "fix".into(),
12781            handover: None,
12782        };
12783        let (_, _, out) = runner
12784            .ask_fixer_among(
12785                attempts.clone(),
12786                "graft",
12787                None,
12788                Some((&winner.worktree, &head)),
12789                build,
12790            )
12791            .await;
12792        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12793        assert!(leftover.exists(), "contention must prevent clean");
12794        assert!(!next.exists());
12795        assert!(runner.state.handovers.is_empty());
12796        // Early interrupted-repair rollback uses the same protected route.
12797        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12798            from_run: "earlier".into(),
12799            generation: 1,
12800            mode: crate::recovery::Mode::VerifyRepair,
12801            reasons: Vec::new(),
12802            reviewed_head: None,
12803            rounds: Vec::new(),
12804            open_findings: Vec::new(),
12805            evidence: None,
12806            criteria_inherited: true,
12807            repair: crate::recovery::Repair::Started {
12808                before: head.clone(),
12809            },
12810        });
12811        assert!(
12812            runner
12813                .recovery_repair(&winner)
12814                .await
12815                .unwrap_err()
12816                .is::<crate::repo_lock::ResourceBlocked>()
12817        );
12818        assert!(matches!(
12819            runner.state.recovery.as_ref().unwrap().repair,
12820            crate::recovery::Repair::Started { .. }
12821        ));
12822        assert!(leftover.exists());
12823        drop(external);
12824        runner.recovery_repair(&winner).await.unwrap();
12825        assert!(!leftover.exists());
12826        assert!(matches!(
12827            runner.state.recovery.as_ref().unwrap().repair,
12828            crate::recovery::Repair::Done { .. }
12829        ));
12830        // A hard lock-open failure also leaves HEAD and untracked work intact.
12831        std::fs::remove_file(&lock_path).unwrap();
12832        std::fs::create_dir(&lock_path).unwrap();
12833        std::fs::write(&leftover, "untrusted edits").unwrap();
12834        let (_, _, out) = runner
12835            .ask_fixer_among(
12836                attempts,
12837                "graft",
12838                None,
12839                Some((&winner.worktree, &head)),
12840                build,
12841            )
12842            .await;
12843        assert!(matches!(out, AgentOutcome::Failed(_)));
12844        assert!(leftover.exists());
12845        assert!(!next.exists());
12846        assert_eq!(
12847            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12848            head
12849        );
12850    }
12851
12852    #[cfg(unix)]
12853    async fn registered_runner(root: &Path) -> Runner {
12854        ask_test_home();
12855        let repo = root.join("repo");
12856        std::fs::create_dir_all(&repo).unwrap();
12857        init_repo(&repo);
12858        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12859        let mut runner = runner_at(RunStatus::Prep);
12860        runner.state.repo = repo.clone();
12861        runner.state.config.coordination = crate::config::test_coordination();
12862        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12863        runner.state.config.graph.worktree_root = Some(root.join("wt"));
12864        runner.state.config.graph.advise = false;
12865        runner.roles.implementers = vec![spec("implement")];
12866        runner.roles.judges = vec![spec("judge")];
12867        runner.prep().await.unwrap();
12868        runner
12869    }
12870
12871    #[tokio::test]
12872    #[cfg(unix)]
12873    async fn parked_implementer_persists_completed_work() {
12874        for registered in [false, true] {
12875            let tmp = tempfile::tempdir().unwrap();
12876            let mut runner = registered_runner(tmp.path()).await;
12877            if !registered {
12878                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12879            }
12880            let marker = tmp.path().join("started");
12881            let release = tmp.path().join("release");
12882            let mut implementer = spec("implement");
12883            implementer.command = vec![
12884                "sh".into(),
12885                "-c".into(),
12886                format!(
12887                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12888                    marker.display(),
12889                    release.display()
12890                ),
12891            ];
12892            runner.roles.implementers = vec![implementer.clone()];
12893            runner.state.config.agents = vec![implementer];
12894            let pause = runner.pause.clone();
12895            let (result, ()) = tokio::join!(runner.implement(), async {
12896                tokio::time::timeout(Duration::from_secs(10), async {
12897                    while !marker.exists() {
12898                        tokio::time::sleep(Duration::from_millis(10)).await;
12899                    }
12900                })
12901                .await
12902                .unwrap();
12903                pause.park();
12904                std::fs::write(&release, "go").unwrap();
12905            });
12906            result.unwrap();
12907            let candidate = &runner.state.candidates[0];
12908            assert!(candidate.commits > 0, "{candidate:?}");
12909            assert!(candidate.summary.contains("saved-result"));
12910            assert_eq!(candidate.files, 1);
12911            let before = candidate.commits;
12912            runner.implement().await.unwrap();
12913            assert_eq!(runner.state.candidates[0].commits, before);
12914        }
12915    }
12916
12917    #[tokio::test]
12918    #[cfg(unix)]
12919    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12920        let tmp = tempfile::tempdir().unwrap();
12921        let mut runner = registered_runner(tmp.path()).await;
12922        let winner = runner.state.candidates[0].clone();
12923        let cache = tmp.path().join("cache");
12924        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12925        runner.state.config.graph.timeout_resource_wait = 0;
12926        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12927        let crate::cache::AcquireOutcome::Acquired(held) =
12928            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12929        else {
12930            panic!("cache must be free")
12931        };
12932        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12933            from_run: "earlier".into(),
12934            generation: 1,
12935            mode: crate::recovery::Mode::VerifyRepair,
12936            reasons: Vec::new(),
12937            reviewed_head: None,
12938            rounds: Vec::new(),
12939            open_findings: Vec::new(),
12940            evidence: None,
12941            criteria_inherited: true,
12942            repair: crate::recovery::Repair::NotStarted,
12943        });
12944        runner.recovery_repair(&winner).await.unwrap();
12945        assert!(matches!(
12946            runner.state.recovery.as_ref().unwrap().repair,
12947            crate::recovery::Repair::WaitingForRestore { .. }
12948        ));
12949        drop(held);
12950        runner.state.resource_block = None;
12951        runner.state.config.graph.timeout_resource_wait = 1;
12952        runner.recovery_repair(&winner).await.unwrap();
12953        assert!(matches!(
12954            runner.state.recovery.as_ref().unwrap().repair,
12955            crate::recovery::Repair::Done { .. }
12956        ));
12957        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12958    }
12959
12960    #[tokio::test]
12961    #[cfg(unix)]
12962    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12963        use std::os::fd::AsRawFd as _;
12964        for graft in [false, true] {
12965            let tmp = tempfile::tempdir().unwrap();
12966            let mut runner = registered_runner(tmp.path()).await;
12967            runner.state.config.graph.timeout_resource_wait = 1;
12968            let winner = runner.state.candidates[0].clone();
12969            let marker = tmp.path().join("fixer-started");
12970            let mut fixer = spec("fixer");
12971            fixer.command = vec![
12972                "sh".into(),
12973                "-c".into(),
12974                format!("echo started >> '{}'; echo done", marker.display()),
12975            ];
12976            runner.roles.fixer = Some(vec![fixer]);
12977            if graft {
12978                let mut loser = winner.clone();
12979                loser.label = 'B';
12980                loser.index = 1;
12981                runner.state.candidates.push(loser);
12982                runner.state.config.graph.graft_runner_up = true;
12983                runner.state.tally = Some(Tally {
12984                    first_choice: BTreeMap::new(),
12985                    borda: BTreeMap::new(),
12986                    winner: winner.label,
12987                    rankings: 1,
12988                    unanimous_initial: true,
12989                    deliberated: false,
12990                    changed_votes: 0,
12991                    unanimous_final: true,
12992                    tie_break: None,
12993                    judges: 1,
12994                    present: 1,
12995                    quorum: 1,
12996                    met_quorum: true,
12997                    uncontested: None,
12998                });
12999                runner.state.judgements.push(crate::run::Judgement {
13000                    judge: 1,
13001                    seat: "judge-1".into(),
13002                    agent: "judge".into(),
13003                    ranking: vec![winner.label, 'B'],
13004                    reasons: BTreeMap::new(),
13005                    confidence: None,
13006                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13007                    order: vec![0, 1],
13008                    failed: None,
13009                    duration_ms: 0,
13010                });
13011            } else {
13012                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13013                    from_run: "earlier".into(),
13014                    generation: 1,
13015                    mode: crate::recovery::Mode::VerifyRepair,
13016                    reasons: Vec::new(),
13017                    reviewed_head: None,
13018                    rounds: Vec::new(),
13019                    open_findings: Vec::new(),
13020                    evidence: None,
13021                    criteria_inherited: true,
13022                    repair: crate::recovery::Repair::NotStarted,
13023                });
13024            }
13025            let reader = crate::repo_lock::SeatLease::for_checkout(
13026                &winner.worktree,
13027                false,
13028                crate::config::test_coordination().as_ref(),
13029            )
13030            .await
13031            .unwrap();
13032            let pause = runner.pause.clone();
13033            let (result, ()) = tokio::join!(
13034                async {
13035                    if graft {
13036                        runner.graft().await
13037                    } else {
13038                        runner.recovery_repair(&winner).await
13039                    }
13040                },
13041                async {
13042                    tokio::time::sleep(Duration::from_millis(100)).await;
13043                    pause.park();
13044                }
13045            );
13046            result.unwrap();
13047            assert!(!marker.exists(), "a blocked seat must not spawn");
13048            // Exercise the persisted state, as a binary replacement/resume does.
13049            runner.state = RunState::load(&runner.state.id).unwrap();
13050            if graft {
13051                assert_eq!(
13052                    runner.state.graft.as_ref().unwrap().outcome,
13053                    GraftOutcome::WaitingForRestore
13054                );
13055            } else {
13056                assert!(matches!(
13057                    runner.state.recovery.as_ref().unwrap().repair,
13058                    crate::recovery::Repair::WaitingForRestore { .. }
13059                ));
13060            }
13061            drop(reader);
13062            runner.pause = Pause::new();
13063            runner.state.resource_block = None;
13064            // Deferred restoration must still respect an external lock owner.
13065            let external =
13066                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13067            assert_eq!(
13068                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13069                0
13070            );
13071            let leftover = winner.worktree.join("leftover");
13072            std::fs::write(&leftover, "earlier fallback edits").unwrap();
13073            let result = if graft {
13074                runner.graft().await
13075            } else {
13076                runner.recovery_repair(&winner).await
13077            };
13078            assert!(
13079                result
13080                    .unwrap_err()
13081                    .is::<crate::repo_lock::ResourceBlocked>()
13082            );
13083            assert!(leftover.exists());
13084            assert!(!marker.exists());
13085            drop(external);
13086            runner.state.resource_block = None;
13087            let result = if graft {
13088                runner.graft().await
13089            } else {
13090                runner.recovery_repair(&winner).await
13091            };
13092            result.unwrap();
13093            assert!(!leftover.exists(), "restore must precede retry");
13094            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13095            if graft {
13096                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13097            } else {
13098                assert!(matches!(
13099                    runner.state.recovery.as_ref().unwrap().repair,
13100                    crate::recovery::Repair::Done { .. }
13101                ));
13102            }
13103        }
13104    }
13105
13106    #[tokio::test]
13107    #[cfg(unix)]
13108    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13109        use std::os::fd::AsRawFd as _;
13110        let tmp = tempfile::tempdir().unwrap();
13111        let mut runner = registered_runner(tmp.path()).await;
13112        runner.state.config.graph.timeout_resource_wait = 1;
13113        let winner = runner.state.candidates[0].clone();
13114        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13115        runner.state.tally = Some(Tally {
13116            first_choice: BTreeMap::new(),
13117            borda: BTreeMap::new(),
13118            winner: winner.label,
13119            rankings: 0,
13120            unanimous_initial: false,
13121            deliberated: false,
13122            changed_votes: 0,
13123            unanimous_final: false,
13124            tie_break: None,
13125            judges: 0,
13126            present: 0,
13127            quorum: 0,
13128            met_quorum: true,
13129            uncontested: Some("fixture".into()),
13130        });
13131        runner.state.graft = Some(GraftRecord {
13132            pre: pre.clone(),
13133            post: None,
13134            lines: None,
13135            limit: None,
13136            agent: None,
13137            outcome: GraftOutcome::Running,
13138            requested: Vec::new(),
13139            adoption: None,
13140            adoption_error: None,
13141        });
13142        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13143        git::commit_all(&winner.worktree, "unfinished graft")
13144            .await
13145            .unwrap();
13146        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13147        let leftover = winner.worktree.join("leftover");
13148        std::fs::write(&leftover, "untrusted edits").unwrap();
13149        let path = runner.state.repo.join(".git/updater.lock");
13150        let external = std::fs::File::create(&path).unwrap();
13151        assert_eq!(
13152            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13153            0
13154        );
13155        assert!(
13156            runner
13157                .graft()
13158                .await
13159                .unwrap_err()
13160                .is::<crate::repo_lock::ResourceBlocked>()
13161        );
13162        assert!(matches!(
13163            runner.state.graft.as_ref().unwrap().outcome,
13164            GraftOutcome::Running
13165        ));
13166        assert_eq!(
13167            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13168            unfinished
13169        );
13170        assert!(leftover.exists());
13171        drop(external);
13172        std::fs::remove_file(&path).unwrap();
13173        std::fs::create_dir(&path).unwrap();
13174        assert!(runner.graft().await.is_err());
13175        assert!(matches!(
13176            runner.state.graft.as_ref().unwrap().outcome,
13177            GraftOutcome::Running
13178        ));
13179        assert_eq!(
13180            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13181            unfinished
13182        );
13183        assert!(leftover.exists());
13184        std::fs::remove_dir(&path).unwrap();
13185        runner.graft().await.unwrap();
13186        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13187        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13188        assert!(!leftover.exists());
13189    }
13190
13191    #[tokio::test]
13192    #[cfg(unix)]
13193    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13194        use std::os::fd::AsRawFd as _;
13195        let tmp = tempfile::tempdir().unwrap();
13196        let mut runner = registered_runner(tmp.path()).await;
13197        runner.state.prep_complete = Some(false);
13198        runner.state.config.graph.timeout_resource_wait = 1;
13199        runner.state.save().unwrap();
13200        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13201        assert_eq!(
13202            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13203            0
13204        );
13205        runner.execute().await.unwrap();
13206        let disk = RunState::load(&runner.state.id).unwrap();
13207        assert!(disk.driver_exited);
13208        assert_eq!(disk.driver_pid, None);
13209        assert_eq!(runner.state.status, RunStatus::Blocked);
13210        assert_eq!(
13211            runner.state.resource_block.as_ref().unwrap().kind,
13212            "repository coordination"
13213        );
13214        assert!(runner.state.prep_failure.is_none());
13215        assert!(runner.state.seats.is_empty());
13216        runner.pause.park();
13217        // Include headroom for Git subprocesses under parallel test load;
13218        // the repository contention deadline itself remains one second.
13219        tokio::time::timeout(Duration::from_secs(10), runner.execute())
13220            .await
13221            .unwrap()
13222            .unwrap();
13223        assert!(runner.state.parked);
13224        assert!(runner.state.prep_failure.is_none());
13225        let disk = RunState::load(&runner.state.id).unwrap();
13226        assert!(disk.driver_exited);
13227        assert_eq!(disk.driver_pid, None);
13228    }
13229
13230    #[tokio::test]
13231    #[cfg(unix)]
13232    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13233        use std::os::fd::AsRawFd as _;
13234        let tmp = tempfile::tempdir().unwrap();
13235        let mut runner = registered_runner(tmp.path()).await;
13236        runner.state.prep_complete = Some(false);
13237        runner.state.config.graph.timeout_resource_wait = 30;
13238        runner.state.save().unwrap();
13239        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13240        assert_eq!(
13241            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13242            0
13243        );
13244        assert!(
13245            tokio::time::timeout(Duration::from_millis(100), runner.execute())
13246                .await
13247                .is_err()
13248        );
13249        let disk = RunState::load(&runner.state.id).unwrap();
13250        assert!(disk.driver_exited);
13251        assert_eq!(disk.driver_pid, None);
13252        assert!(disk.status.resumable());
13253        assert!(runner.state.driver_exited);
13254        assert_eq!(runner.state.driver_pid, None);
13255        runner.state.save().unwrap();
13256        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13257    }
13258
13259    #[tokio::test]
13260    #[cfg(unix)]
13261    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13262        let tmp = tempfile::tempdir().unwrap();
13263        let mut runner = registered_runner(tmp.path()).await;
13264        let winner = runner.state.candidates[0].clone();
13265        let cache = tmp.path().join("cache");
13266        let started = tmp.path().join("pre-gate-started");
13267        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13268        runner.state.config.verify.gate =
13269            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13270        let home = crate::run::home();
13271        let owner =
13272            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13273        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13274            crate::cache::AcquireOutcome::Acquired(g) => g,
13275            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13276        };
13277        let mut gate = Box::pin(runner.run_gate(&winner));
13278        assert!(
13279            tokio::time::timeout(Duration::from_millis(150), &mut gate)
13280                .await
13281                .is_err()
13282        );
13283        assert!(!started.exists(), "pre-gate must wait for the cache too");
13284        // Simulate the wave that already owns the cache entering invoke.
13285        // It must obtain the editor reservation and finish without waiting
13286        // for the gate that is queued behind its cache.
13287        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13288        let editor = crate::repo_lock::SeatLease::checkout_with(
13289            &winner.worktree,
13290            true,
13291            crate::config::test_coordination().as_ref(),
13292            &wait,
13293        )
13294        .await
13295        .unwrap();
13296        drop(editor);
13297        guard.release();
13298        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13299            .await
13300            .unwrap()
13301            .unwrap();
13302        drop(gate);
13303        assert!(outcomes.iter().all(CommandOutcome::ok));
13304        assert!(started.exists());
13305    }
13306
13307    #[tokio::test]
13308    #[cfg(unix)]
13309    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13310        use std::os::fd::AsRawFd as _;
13311
13312        ask_test_home();
13313        let tmp = tempfile::tempdir().unwrap();
13314        let repo = tmp.path().join("repo");
13315        std::fs::create_dir_all(&repo).unwrap();
13316        init_repo(&repo);
13317        let common = repo.join(".git");
13318        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13319        let mut runner = runner_at(RunStatus::Prep);
13320        runner.state.repo = repo.clone();
13321        runner.state.config.coordination = crate::config::test_coordination();
13322        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13323        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13324        runner.state.config.graph.advise = false;
13325        runner.roles.implementers = vec![spec("implement")];
13326        runner.roles.judges = vec![spec("judge")];
13327        runner.prep().await.unwrap();
13328        let winner = runner.state.candidates[0].clone();
13329        // Emulate edits already left by a completed seat. The fixture CLI
13330        // only reports completion; it performs no repository operations.
13331        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13332        let completed = tmp.path().join("seat-completed");
13333        runner.roles.implementers[0].command = vec![
13334            "sh".into(),
13335            "-c".into(),
13336            format!("touch '{}'; echo completed", completed.display()),
13337        ];
13338        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13339        assert_eq!(
13340            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13341            0
13342        );
13343        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13344        let mut implement = Box::pin(runner.implement());
13345        tokio::select! {
13346            result = &mut implement => panic!("rescue must wait: {result:?}"),
13347            result = tokio::time::timeout(Duration::from_secs(10), async {
13348                while !completed.exists() {
13349                    tokio::time::sleep(Duration::from_millis(20)).await;
13350                }
13351            }) => result.unwrap(),
13352        }
13353        assert!(
13354            tokio::time::timeout(Duration::from_millis(200), &mut implement)
13355                .await
13356                .is_err()
13357        );
13358        assert_eq!(
13359            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13360            before
13361        );
13362        assert_eq!(
13363            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13364            0
13365        );
13366        tokio::time::timeout(Duration::from_secs(10), &mut implement)
13367            .await
13368            .unwrap()
13369            .unwrap();
13370        drop(implement);
13371        assert_ne!(
13372            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13373            before
13374        );
13375        assert!(git::is_clean(&winner.worktree).await.unwrap());
13376
13377        let started = tmp.path().join("pre-gate-started");
13378        let proceed = tmp.path().join("pre-gate-proceed");
13379        let checked = tmp.path().join("gate-ran");
13380        runner.state.config.verify.pre_gate = vec![format!(
13381            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13382            started.display(),
13383            proceed.display()
13384        )];
13385        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13386        assert_eq!(
13387            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13388            0
13389        );
13390        let mut gate = Box::pin(runner.run_gate(&winner));
13391        assert!(
13392            tokio::time::timeout(Duration::from_millis(200), &mut gate)
13393                .await
13394                .is_err()
13395        );
13396        assert!(!started.exists());
13397        assert!(!checked.exists());
13398        assert!(git::is_clean(&winner.worktree).await.unwrap());
13399        assert_eq!(
13400            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13401            0
13402        );
13403        tokio::select! {
13404            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13405            result = tokio::time::timeout(Duration::from_secs(10), async {
13406                while !started.exists() {
13407                    tokio::time::sleep(Duration::from_millis(20)).await;
13408                }
13409            }) => result.unwrap(),
13410        }
13411        assert_eq!(
13412            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13413            -1,
13414            "mechanical edits must hold the real lock"
13415        );
13416        std::fs::write(proceed, "ready").unwrap();
13417        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13418            .await
13419            .unwrap()
13420            .unwrap();
13421        drop(gate);
13422        assert!(outcomes.iter().all(CommandOutcome::ok));
13423        assert!(checked.exists());
13424        assert!(runner.state.pre_gate_commit.is_some());
13425        assert!(git::is_clean(&winner.worktree).await.unwrap());
13426        assert_eq!(
13427            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13428            0,
13429            "gate completion must release coordination"
13430        );
13431        drop(outsider);
13432        std::fs::remove_file(common.join("updater.lock")).unwrap();
13433        std::fs::create_dir(common.join("updater.lock")).unwrap();
13434        std::fs::remove_file(&started).unwrap();
13435        std::fs::remove_file(&checked).unwrap();
13436        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13437        let error = runner.run_gate(&winner).await.unwrap_err();
13438        assert!(
13439            error
13440                .to_string()
13441                .contains("acquire mandatory repository lock")
13442        );
13443        assert!(
13444            !started.exists(),
13445            "failed acquisition must prevent pre-gate"
13446        );
13447        assert!(!checked.exists(), "failed acquisition must prevent gate");
13448        assert!(git::is_clean(&winner.worktree).await.unwrap());
13449        assert_eq!(
13450            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13451            head
13452        );
13453    }
13454
13455    #[tokio::test]
13456    #[cfg(unix)]
13457    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13458        use std::os::fd::AsRawFd as _;
13459        use std::time::Duration;
13460
13461        ask_test_home();
13462        let tmp = tempfile::tempdir().unwrap();
13463        let repo = tmp.path().join("repo");
13464        std::fs::create_dir_all(&repo).unwrap();
13465        init_repo(&repo);
13466        let common = repo.join(".git");
13467        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13468        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13469        assert_eq!(
13470            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13471            0
13472        );
13473        let mut runner = runner_at(RunStatus::Prep);
13474        runner.state.repo = repo.clone();
13475        runner.state.config.coordination = crate::config::test_coordination();
13476        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13477        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13478        runner.state.config.graph.advise = true;
13479        runner.state.config.graph.advisors = 1;
13480        runner.roles.implementers = vec![spec("implement")];
13481        runner.roles.judges = vec![spec("judge")];
13482        let root = runner.state.worktree_root();
13483        let started = tmp.path().join("setup-started");
13484        let proceed = tmp.path().join("setup-proceed");
13485        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13486            run: Some(format!(
13487                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13488                started.display(),
13489                proceed.display()
13490            )),
13491            ..Default::default()
13492        }];
13493        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13494        let config = git::git(&repo, &["config", "--local", "--list"])
13495            .await
13496            .unwrap();
13497        let mut prep = Box::pin(runner.prep());
13498        assert!(
13499            tokio::time::timeout(Duration::from_millis(200), &mut prep)
13500                .await
13501                .is_err()
13502        );
13503        assert!(
13504            !root.exists(),
13505            "contention must prevent even checkout creation"
13506        );
13507        assert!(!started.exists(), "contention must prevent setup");
13508        assert_eq!(
13509            git::git(&repo, &["branch", "--list"]).await.unwrap(),
13510            branches
13511        );
13512        assert_eq!(
13513            git::git(&repo, &["config", "--local", "--list"])
13514                .await
13515                .unwrap(),
13516            config
13517        );
13518        drop(lock);
13519        tokio::select! {
13520            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13521            result = tokio::time::timeout(Duration::from_secs(10), async {
13522                while !started.exists() {
13523                    tokio::time::sleep(Duration::from_millis(20)).await;
13524                }
13525            }) => result.unwrap(),
13526        }
13527        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13528        assert_eq!(
13529            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13530            -1
13531        );
13532        assert!(
13533            tokio::time::timeout(
13534                Duration::from_millis(100),
13535                crate::repo_lock::SeatLease::for_checkout(
13536                    &repo,
13537                    true,
13538                    crate::config::test_coordination().as_ref()
13539                )
13540            )
13541            .await
13542            .is_err(),
13543            "preparation must also exclude local editors"
13544        );
13545        std::fs::write(proceed, "ready").unwrap();
13546        tokio::time::timeout(Duration::from_secs(10), &mut prep)
13547            .await
13548            .unwrap()
13549            .unwrap();
13550        drop(prep);
13551        assert_eq!(runner.state.prep_complete, Some(true));
13552        assert!(root.join("judge-1").exists());
13553        assert!(root.join("advisor-1").exists());
13554        assert_eq!(
13555            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13556            0
13557        );
13558    }
13559
13560    #[tokio::test]
13561    #[cfg(unix)]
13562    async fn preparation_lock_failure_prevents_repository_changes() {
13563        ask_test_home();
13564        let tmp = tempfile::tempdir().unwrap();
13565        let repo = tmp.path().join("repo");
13566        std::fs::create_dir_all(&repo).unwrap();
13567        init_repo(&repo);
13568        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13569        // A directory at the required lock path makes opening the real lock
13570        // fail, rather than merely exercising an invalid registration.
13571        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13572        let mut runner = runner_at(RunStatus::Prep);
13573        runner.state.repo = repo.clone();
13574        runner.state.config.coordination = crate::config::test_coordination();
13575        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13576        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13577        runner.roles.implementers = vec![spec("implement")];
13578        let error = runner.prep().await.unwrap_err();
13579        assert!(
13580            error
13581                .to_string()
13582                .contains("acquire mandatory repository lock")
13583        );
13584        assert!(!runner.state.worktree_root().exists());
13585        assert!(runner.state.candidates.is_empty());
13586        assert!(!repo.join(".git/config.worktree").exists());
13587        assert_eq!(
13588            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13589            "* main"
13590        );
13591    }
13592
13593    #[tokio::test]
13594    async fn preparation_records_creation_intent_before_git_can_fail() {
13595        ask_test_home();
13596        let tmp = tempfile::tempdir().unwrap();
13597        let repo = tmp.path().join("repo");
13598        std::fs::create_dir_all(&repo).unwrap();
13599        init_repo(&repo);
13600        let mut runner = runner_at(RunStatus::Prep);
13601        runner.state.repo = repo.clone();
13602        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13603        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13604        runner.state.config.blind.commit_msg_hook = false;
13605        runner.roles.implementers = vec![spec("one")];
13606        let label = blind::assign_labels(1, runner.state.seed)[0];
13607        let branch = runner.state.branch_for(label);
13608        // A conflicting branch makes Git refuse creation before a candidate
13609        // can be registered. The durable intent must still identify it.
13610        git::git(&repo, &["branch", &branch]).await.unwrap();
13611        assert!(runner.prep().await.is_err());
13612        let saved = RunState::load(&runner.state.id).unwrap();
13613        assert!(saved.candidates.is_empty());
13614        let event = saved
13615            .events
13616            .iter()
13617            .find(|e| e.node == "prep-intent")
13618            .unwrap();
13619        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13620        assert_eq!(intent["branch"], branch);
13621        assert_eq!(intent["start"], runner.state.base_commit);
13622        assert_eq!(
13623            intent["worktree"],
13624            runner
13625                .state
13626                .worktree_root()
13627                .join(format!("cand-{label}"))
13628                .to_string_lossy()
13629                .as_ref()
13630        );
13631        assert_eq!(intent["picks"], serde_json::json!([]));
13632    }
13633
13634    #[tokio::test]
13635    async fn invalid_pinned_sources_create_no_partial_worktrees() {
13636        ask_test_home();
13637        let tmp = tempfile::tempdir().unwrap();
13638        let repo = tmp.path().join("repo");
13639        std::fs::create_dir_all(&repo).unwrap();
13640        init_repo(&repo);
13641        let mut runner = runner_at(RunStatus::Prep);
13642        runner.state.repo = repo.clone();
13643        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13644        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13645        runner.state.sources.pick_commits.push(refs::Input {
13646            token: "missing".into(),
13647            sha: "0".repeat(40),
13648            provenance: refs::Provenance::ExplicitCli,
13649        });
13650        assert!(runner.prep().await.is_err());
13651        assert!(runner.state.candidates.is_empty());
13652        assert!(!runner.state.worktree_root().exists());
13653    }
13654
13655    #[tokio::test]
13656    async fn prose_and_legacy_inference_never_import_into_candidates() {
13657        ask_test_home();
13658        let tmp = tempfile::tempdir().unwrap();
13659        let repo = tmp.path().join("repo");
13660        std::fs::create_dir_all(&repo).unwrap();
13661        init_repo(&repo);
13662        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13663        git::git(&repo, &["checkout", "-b", "magi/history/A"])
13664            .await
13665            .unwrap();
13666        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13667        git::git(&repo, &["add", "."]).await.unwrap();
13668        git::git(&repo, &["commit", "-m", "historical implementation"])
13669            .await
13670            .unwrap();
13671        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13672        git::git(&repo, &["checkout", "main"]).await.unwrap();
13673        for instruction in [
13674            format!("Example: magi/history/A at {source}"),
13675            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13676            format!("Never import magi/history/A or {source}"),
13677            format!("Conversation answer: I mentioned {source} only as background"),
13678        ] {
13679            let mut runner = runner_at(RunStatus::Prep);
13680            runner.state.repo = repo.clone();
13681            runner.state.base_commit = base.clone();
13682            runner.state.instruction = instruction.clone();
13683            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13684            runner.state.config.blind.commit_msg_hook = false;
13685            runner.state.seeds = vec![refs::Seed {
13686                token: "magi/history/A".into(),
13687                sha: source.clone(),
13688                kind: refs::SeedKind::Unmerged,
13689                branch: true,
13690                detail: String::new(),
13691            }];
13692            runner.roles.implementers = vec![spec("one")];
13693            assert_eq!(seeded_instruction(&runner.state), instruction);
13694            // Model an old record with inferred seeds but no structured input field.
13695            let mut legacy = serde_json::to_value(&runner.state).unwrap();
13696            legacy.as_object_mut().unwrap().remove("sources");
13697            runner.state = serde_json::from_value(legacy).unwrap();
13698            assert_eq!(runner.state.sources, refs::Sources::default());
13699            runner.state.save().unwrap();
13700            runner.state = RunState::load(&runner.state.id).unwrap();
13701            runner.prep().await.unwrap();
13702            for candidate in &runner.state.candidates {
13703                assert_eq!(
13704                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13705                    base
13706                );
13707                assert!(!candidate.worktree.join("unwanted").exists());
13708            }
13709        }
13710    }
13711
13712    #[tokio::test]
13713    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13714        ask_test_home();
13715        let tmp = tempfile::tempdir().unwrap();
13716        let repo = tmp.path().join("repo");
13717        std::fs::create_dir_all(&repo).unwrap();
13718        init_repo(&repo);
13719        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13720        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13721        std::fs::write(repo.join("referenced"), "required commit").unwrap();
13722        git::git(&repo, &["add", "."]).await.unwrap();
13723        git::git(&repo, &["commit", "-m", "required seed"])
13724            .await
13725            .unwrap();
13726        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13727        git::git(&repo, &["checkout", "main"]).await.unwrap();
13728        let mut runner = runner_at(RunStatus::Prep);
13729        runner.state.repo = repo.clone();
13730        runner.state.base_commit = base.clone();
13731        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13732        runner.state.config.graph.advise = false;
13733        runner.state.config.blind.commit_msg_hook = false;
13734        runner.state.sources =
13735            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13736                .await
13737                .unwrap();
13738        runner.roles.implementers = vec![spec("one"), spec("two")];
13739        runner.roles.judges = vec![spec("judge")];
13740        let label = blind::assign_labels(2, runner.state.seed)[0];
13741        let branch = runner.state.branch_for(label);
13742        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13743        git::worktree_add_branch(&repo, &worktree, &branch, &base)
13744            .await
13745            .unwrap();
13746        runner.state.candidates.push(Candidate {
13747            index: 0,
13748            label,
13749            agent: "one".into(),
13750            branch,
13751            worktree,
13752            summary: String::new(),
13753            stat: String::new(),
13754            files: 0,
13755            commits: 0,
13756            empty: false,
13757            failed: None,
13758            verified_noop: None,
13759            duration_ms: 0,
13760            folded: false,
13761        });
13762        runner.state.save().unwrap();
13763        // Reload the checkpoint immediately before its first cherry-pick.
13764        runner.state = RunState::load(&runner.state.id).unwrap();
13765        assert_eq!(runner.state.prep_complete, Some(false));
13766        runner.prep().await.unwrap();
13767        assert_eq!(runner.state.prep_complete, Some(true));
13768        assert_eq!(runner.state.candidates.len(), 2);
13769        for candidate in &runner.state.candidates {
13770            assert_eq!(
13771                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13772                git::tree_of(&repo, &seed).await.unwrap()
13773            );
13774            assert_eq!(
13775                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13776                "required commit"
13777            );
13778        }
13779        assert!(runner.state.worktree_root().join("judge-1").exists());
13780        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13781        runner.prep().await.unwrap();
13782        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13783
13784        runner.state.prep_complete = Some(false);
13785        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13786            run: Some(format!(
13787                "test -f '{}'",
13788                tmp.path().join("dependency-ready").display()
13789            )),
13790            ..Default::default()
13791        }];
13792        assert!(runner.prep().await.is_err());
13793        assert!(runner.state.prep_failure.is_some());
13794        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13795        runner.state = RunState::load(&runner.state.id).unwrap();
13796        assert!(
13797            runner.prep().await.is_err(),
13798            "correcting the cause alone must not auto-retry"
13799        );
13800        let failed_at = runner.state.events.last().unwrap().at;
13801        assert!(
13802            runner.retry_preparation(failed_at).is_err(),
13803            "old approvals cannot unlock the hold"
13804        );
13805        let mut task = crate::queue::Task::new(
13806            "retry".into(),
13807            "retry setup".into(),
13808            repo.clone(),
13809            crate::queue::Source::Human,
13810        );
13811        task.start(runner.state.id.clone());
13812        task.hold_machine(Some("setup failed".into()));
13813        task.release_operator("cli", None);
13814        runner
13815            .retry_preparation(task.resumption.as_ref().unwrap().at)
13816            .unwrap();
13817        runner.prep().await.unwrap();
13818        assert!(runner.state.prep_failure.is_none());
13819        assert_eq!(runner.state.prep_complete, Some(true));
13820        assert_eq!(runner.state.candidates.len(), 2);
13821        assert!(runner.state.worktree_root().join("judge-1").exists());
13822        for candidate in &runner.state.candidates {
13823            assert_eq!(
13824                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13825                git::tree_of(&repo, &seed).await.unwrap()
13826            );
13827        }
13828    }
13829
13830    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13831        let mut heads = Vec::new();
13832        for candidate in candidates {
13833            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13834        }
13835        heads
13836    }
13837
13838    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13839        ask_test_home();
13840        let tmp = tempfile::tempdir().unwrap();
13841        let repo = tmp.path().join("repo");
13842        std::fs::create_dir(&repo).unwrap();
13843        init_repo(&repo);
13844        let mut runner = runner_at(RunStatus::Reviewing);
13845        runner.state.repo = repo.clone();
13846        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13847        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13848        for (index, label) in ['A', 'B'].into_iter().enumerate() {
13849            let branch = format!("candidate-{label}");
13850            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13851            git::worktree_add_branch(&repo, &worktree, &branch, "main")
13852                .await
13853                .unwrap();
13854            if label == 'B' {
13855                std::fs::write(worktree.join("change"), "winner").unwrap();
13856                git::rescue_commit(&worktree, "candidate change")
13857                    .await
13858                    .unwrap();
13859            }
13860            runner.state.candidates.push(Candidate {
13861                index,
13862                label,
13863                agent: "alpha".into(),
13864                branch,
13865                worktree,
13866                summary: "completed reply".into(),
13867                stat: String::new(),
13868                files: usize::from(label == 'B'),
13869                commits: usize::from(label == 'B'),
13870                empty: label == 'A',
13871                failed: None,
13872                verified_noop: None,
13873                duration_ms: 1,
13874                folded: false,
13875            });
13876        }
13877        runner.state.judge_skipped = true;
13878        runner.state.tally = Some(Tally {
13879            first_choice: BTreeMap::new(),
13880            borda: BTreeMap::new(),
13881            winner: 'B',
13882            rankings: 0,
13883            unanimous_initial: false,
13884            deliberated: false,
13885            changed_votes: 0,
13886            unanimous_final: false,
13887            tie_break: None,
13888            judges: 0,
13889            present: 0,
13890            quorum: 0,
13891            met_quorum: true,
13892            uncontested: Some("only B produced a change".into()),
13893        });
13894        runner.state.save().unwrap();
13895        (tmp, runner)
13896    }
13897
13898    #[tokio::test]
13899    #[cfg(unix)]
13900    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13901        for (solo, failed_checkpoint) in
13902            [(true, false), (false, false), (true, true), (false, true)]
13903        {
13904            let (_tmp, mut runner) = preservation_resume_fixture().await;
13905            if solo {
13906                runner.state.candidates.retain(|c| c.label == 'A');
13907            }
13908            runner.state.candidates[0].empty = false;
13909            runner.state.candidates[0].summary.clear();
13910            runner.state.tally = None;
13911            runner.state.judge_skipped = false;
13912            runner.state.status = RunStatus::Implementing;
13913            runner.state.config.disk.min_free_bytes = u64::MAX;
13914            let mut implementer = spec("alpha");
13915            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13916            runner.roles.implementers = vec![implementer];
13917            let patch = runner.state.dir().join("artifacts/cand-A.patch");
13918            if failed_checkpoint {
13919                std::fs::create_dir_all(&patch).unwrap();
13920            }
13921            runner.state.save().unwrap();
13922            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13923            runner.state = RunState::load(&runner.state.id).unwrap();
13924            assert_eq!(runner.state.status, RunStatus::Blocked);
13925            assert!(runner.state.interrupted_implementations.contains(&'A'));
13926            assert_eq!(runner.state.candidates[0].commits, 0);
13927            assert!(!runner.state.candidates[0].empty);
13928            assert!(runner.state.candidates[0].failed.is_none());
13929            if failed_checkpoint {
13930                assert!(runner.state.candidate_preservation.contains_key(&'A'));
13931                std::fs::remove_dir(patch).unwrap();
13932            }
13933            runner.state.resource_block = None;
13934            runner.state.config.disk.min_free_bytes = 0;
13935            runner.roles.implementers[0].command = vec![
13936                "sh".into(),
13937                "-c".into(),
13938                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13939            ];
13940            runner.implement().await.unwrap();
13941            assert!(runner.state.interrupted_implementations.is_empty());
13942            assert!(runner.state.candidate_preservation.is_empty());
13943            assert_eq!(runner.state.candidates[0].commits, 1);
13944            assert!(!runner.state.candidates[0].empty);
13945            assert!(
13946                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13947                    .unwrap()
13948                    .contains("retried empty interrupted seat")
13949            );
13950            if !solo {
13951                assert_eq!(runner.state.candidates[1].commits, 1);
13952            }
13953        }
13954    }
13955
13956    #[tokio::test]
13957    #[cfg(unix)]
13958    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
13959        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
13960            let (_tmp, mut runner) = preservation_resume_fixture().await;
13961            runner.state.candidates.retain(|c| c.label == 'A');
13962            runner.state.candidates[0].empty = false;
13963            runner.state.candidates[0].summary.clear();
13964            runner.state.tally = None;
13965            runner.state.judge_skipped = false;
13966            runner.state.status = RunStatus::Implementing;
13967            // Force the real post-spawn watchdog, independently of available
13968            // disk space. Dirty/committed work predates its first measurement.
13969            runner.state.config.disk.min_free_bytes = u64::MAX;
13970            let wt = runner.state.candidates[0].worktree.clone();
13971            std::fs::write(
13972                wt.join("interrupted-source"),
13973                "retain implementation work\n",
13974            )
13975            .unwrap();
13976            if committed {
13977                git::rescue_commit(&wt, "implementation checkpoint")
13978                    .await
13979                    .unwrap();
13980            }
13981            let lock = git::git(
13982                &wt,
13983                &[
13984                    "rev-parse",
13985                    "--path-format=absolute",
13986                    "--git-path",
13987                    "index.lock",
13988                ],
13989            )
13990            .await
13991            .unwrap();
13992            if block_rescue {
13993                std::fs::write(&lock, "fixture-owned lock").unwrap();
13994            }
13995            let mut implementer = spec("alpha");
13996            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13997            runner.roles.implementers = vec![implementer];
13998            runner.state.save().unwrap();
13999            let result = runner.implement().await;
14000            assert_eq!(result.is_err(), block_rescue);
14001            assert_eq!(
14002                runner.state.resource_block.as_ref().unwrap().kind,
14003                "disk pressure"
14004            );
14005            assert!(
14006                runner
14007                    .state
14008                    .events
14009                    .iter()
14010                    .any(|e| e.message.contains("agent execution interrupted"))
14011            );
14012            assert!(
14013                runner
14014                    .state
14015                    .dir()
14016                    .join("artifacts/candidate-A-recovery/staged.patch")
14017                    .exists()
14018            );
14019            if block_rescue {
14020                let saved = RunState::load(&runner.state.id).unwrap();
14021                assert!(saved.candidate_preservation.contains_key(&'A'));
14022                assert!(!saved.candidates[0].empty);
14023                assert_eq!(
14024                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14025                    "retain implementation work\n"
14026                );
14027                std::fs::remove_file(lock).unwrap();
14028                runner.state = saved;
14029            } else {
14030                assert_eq!(runner.state.status, RunStatus::Blocked);
14031                assert_eq!(runner.state.candidates[0].commits, 1);
14032            }
14033            // Recovery must save the candidate before deciding whether another
14034            // implementation is needed. No implementer is available on retry.
14035            runner.roles.implementers.clear();
14036            runner.state.config.disk.min_free_bytes = 0;
14037            runner.state.resource_block = None;
14038            runner.implement().await.unwrap();
14039            assert!(runner.state.candidate_preservation.is_empty());
14040            assert_eq!(runner.state.candidates[0].commits, 1);
14041            assert!(!runner.state.candidates[0].empty);
14042            assert!(git::is_clean(&wt).await.unwrap());
14043            assert!(
14044                crate::run::read_artifact(&runner.state, "cand-A.patch")
14045                    .unwrap()
14046                    .contains("retain implementation work")
14047            );
14048        }
14049    }
14050
14051    #[tokio::test]
14052    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14053        let (_tmp, mut runner) = preservation_resume_fixture().await;
14054        // Schema 15 already confirmed the unfolded no-change result.
14055        runner.implement().await.unwrap();
14056        assert!(runner.state.candidate_preservation.is_empty());
14057        assert!(
14058            !runner
14059                .state
14060                .dir()
14061                .join("artifacts/candidate-A-recovery")
14062                .exists()
14063        );
14064        runner.fold_losers().await.unwrap();
14065        assert!(!runner.state.candidates[0].worktree.exists());
14066        for schema in [14, crate::run::SCHEMA] {
14067            runner.state.schema = schema;
14068            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14069            runner.state.save().unwrap();
14070            runner.state = RunState::load(&runner.state.id).unwrap();
14071            runner.implement().await.unwrap();
14072            assert!(runner.state.candidate_preservation.is_empty());
14073            assert!(runner.state.candidates[0].folded);
14074            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14075        }
14076    }
14077
14078    #[tokio::test]
14079    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14080        let (_tmp, mut runner) = preservation_resume_fixture().await;
14081        let wt = runner.state.candidates[0].worktree.clone();
14082        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14083        runner.state.schema = 14;
14084        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14085        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14086        let tally = serde_json::to_value(&runner.state.tally).unwrap();
14087        for _ in 0..2 {
14088            assert!(runner.implement().await.is_err());
14089            assert_eq!(runner.state.status, RunStatus::Blocked);
14090            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14091            assert!(
14092                runner.state.candidates[0].empty,
14093                "old verdict must remain unusable"
14094            );
14095            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14096            assert!(runner.state.judge_skipped);
14097            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14098            assert!(
14099                crate::run::read_artifact(&runner.state, "cand-A.patch")
14100                    .unwrap()
14101                    .contains("owned edits")
14102            );
14103            assert!(wt.exists());
14104            assert_eq!(
14105                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14106                    .await
14107                    .unwrap(),
14108                1
14109            );
14110            runner.state = RunState::load(&runner.state.id).unwrap();
14111        }
14112    }
14113
14114    #[tokio::test]
14115    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14116        let (_tmp, mut runner) = preservation_resume_fixture().await;
14117        let wt = runner.state.candidates[0].worktree.clone();
14118        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14119        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14120        let index_lock = PathBuf::from(
14121            git::git(
14122                &wt,
14123                &[
14124                    "rev-parse",
14125                    "--path-format=absolute",
14126                    "--git-path",
14127                    "index.lock",
14128                ],
14129            )
14130            .await
14131            .unwrap(),
14132        );
14133        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14134        runner.state.schema = 14;
14135        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14136        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14137        assert!(runner.implement().await.is_err());
14138        assert!(runner.after_implement().is_err());
14139        assert!(runner.fold_losers().await.is_err());
14140        assert_eq!(
14141            std::fs::read_to_string(wt.join("recovered")).unwrap(),
14142            "failed seat edits"
14143        );
14144        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14145        runner.state = RunState::load(&runner.state.id).unwrap();
14146        std::fs::remove_file(index_lock).unwrap();
14147        // Recovery finds changes that the old uncontested result never judged.
14148        assert!(runner.implement().await.is_err());
14149        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14150        assert!(runner.state.candidates[0].empty);
14151        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14152        assert!(
14153            crate::run::read_artifact(&runner.state, "cand-A.patch")
14154                .unwrap()
14155                .contains("failed seat edits")
14156        );
14157        assert!(wt.exists());
14158    }
14159
14160    #[tokio::test]
14161    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14162        let (_tmp, mut runner) = preservation_resume_fixture().await;
14163        let repo = runner.state.repo.clone();
14164        let common = git::git(
14165            &repo,
14166            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14167        )
14168        .await
14169        .unwrap();
14170        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14171        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14172        assert!(runner.fold_losers().await.is_err());
14173        assert!(!runner.state.candidates[0].worktree.exists());
14174        assert!(!runner.state.candidates[0].folded);
14175        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14176        runner.state = RunState::load(&runner.state.id).unwrap();
14177        std::fs::remove_file(branch_lock).unwrap();
14178        let judge = runner.state.worktree_root().join("judge-1");
14179        git::worktree_add_detached(&repo, &judge, "main")
14180            .await
14181            .unwrap();
14182        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14183        runner.roles.judges = vec![spec("judge")];
14184        assert!(runner.fold_losers().await.is_err());
14185        runner.state = RunState::load(&runner.state.id).unwrap();
14186        assert!(
14187            runner.state.candidates[0].folded,
14188            "candidate progress saved before judge failure"
14189        );
14190        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14191        assert!(judge.join("uncommitted").exists());
14192        std::fs::remove_file(judge.join("uncommitted")).unwrap();
14193        runner.fold_losers().await.unwrap();
14194        assert!(!judge.exists());
14195        runner.fold_losers().await.unwrap();
14196    }
14197
14198    #[tokio::test]
14199    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14200        let (_tmp, mut runner) = preservation_resume_fixture().await;
14201        runner.state.candidates.retain(|c| c.label == 'B');
14202        runner.state.tally = None;
14203        runner.state.judge_skipped = false;
14204        runner.state.prep_complete = Some(true);
14205        runner.state.config.graph.advise = false;
14206        runner.state.config.graph.selected_implementer = Some("alpha".into());
14207        let wt = runner.state.candidates[0].worktree.clone();
14208        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14209        runner
14210            .state
14211            .candidate_preservation
14212            .insert('B', "awaiting persistence".into());
14213        let index_lock = git::git(
14214            &wt,
14215            &[
14216                "rev-parse",
14217                "--path-format=absolute",
14218                "--git-path",
14219                "index.lock",
14220            ],
14221        )
14222        .await
14223        .unwrap();
14224        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14225        runner.state.save().unwrap();
14226        assert!(runner.execute_graph().await.is_err());
14227        assert_eq!(runner.state.status, RunStatus::Blocked);
14228        assert!(runner.state.candidate_preservation.contains_key(&'B'));
14229        std::fs::remove_file(index_lock).unwrap();
14230        // Park at the next graph boundary: this tests the real selected-seat
14231        // hold guard without dispatching reviewers or publishing anything.
14232        for _ in 0..2 {
14233            runner.state = RunState::load(&runner.state.id).unwrap();
14234            runner.pause.park();
14235            runner.execute_graph().await.unwrap();
14236            assert_eq!(runner.state.status, RunStatus::Implementing);
14237            assert!(runner.state.candidate_preservation.is_empty());
14238            assert_eq!(runner.state.candidates[0].commits, 2);
14239            assert!(runner.state.tally.is_none());
14240        }
14241        runner.implement().await.unwrap();
14242        assert_eq!(runner.state.status, RunStatus::Judging);
14243        // A genuine selected-implementer hold must still require release.
14244        runner.state.status = RunStatus::Blocked;
14245        runner.state.candidates[0].failed = Some("implementation failed".into());
14246        runner.state.save().unwrap();
14247        let error = runner.execute_graph().await.unwrap_err();
14248        assert!(
14249            error
14250                .to_string()
14251                .contains("selected implementation is held")
14252        );
14253    }
14254
14255    #[tokio::test]
14256    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14257        let (_tmp, mut runner) = preservation_resume_fixture().await;
14258        let repo = runner.state.repo.clone();
14259        let candidate = runner.state.candidates[0].clone();
14260        // Remove only the disposable fixture directory, leaving Git's linked
14261        // worktree registration and branch intact, as in a vanished checkout.
14262        let registered = git::worktree_holding(&repo, &candidate.branch)
14263            .await
14264            .unwrap()
14265            .unwrap();
14266        assert_eq!(
14267            registered.canonicalize().unwrap(),
14268            candidate.worktree.canonicalize().unwrap()
14269        );
14270        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14271        assert_eq!(
14272            git::worktree_holding(&repo, &candidate.branch)
14273                .await
14274                .unwrap(),
14275            Some(registered)
14276        );
14277        assert!(
14278            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14279            "the stale registration initially keeps the branch checked out"
14280        );
14281        runner.fold_losers().await.unwrap();
14282        assert!(runner.state.candidates[0].folded);
14283        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14284        assert_eq!(
14285            git::worktree_holding(&repo, &candidate.branch)
14286                .await
14287                .unwrap(),
14288            None
14289        );
14290        runner.state = RunState::load(&runner.state.id).unwrap();
14291        runner.fold_losers().await.unwrap();
14292    }
14293
14294    #[tokio::test]
14295    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14296        let (_tmp, mut runner) = preservation_resume_fixture().await;
14297        let repo = runner.state.repo.clone();
14298        let candidate = runner.state.candidates[0].clone();
14299        let winner = runner.state.candidates[1].clone();
14300        let registered = git::worktree_holding(&repo, &candidate.branch)
14301            .await
14302            .unwrap()
14303            .unwrap();
14304        assert_eq!(
14305            registered.canonicalize().unwrap(),
14306            candidate.worktree.canonicalize().unwrap()
14307        );
14308        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14309        assert_eq!(
14310            git::worktree_holding(&repo, &candidate.branch)
14311                .await
14312                .unwrap(),
14313            Some(registered)
14314        );
14315        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14316
14317        for _ in 0..2 {
14318            fold_run(&mut runner.state, false, &crate::run::home())
14319                .await
14320                .unwrap();
14321            runner.state = RunState::load(&runner.state.id).unwrap();
14322            assert!(runner.state.candidates[0].folded);
14323            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14324            assert_eq!(
14325                git::worktree_holding(&repo, &candidate.branch)
14326                    .await
14327                    .unwrap(),
14328                None
14329            );
14330            assert!(!runner.state.candidates[1].folded);
14331            assert!(winner.worktree.exists());
14332            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14333        }
14334    }
14335
14336    #[tokio::test]
14337    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14338        ask_test_home();
14339        let tmp = tempfile::tempdir().unwrap();
14340        let repo = tmp.path().join("repo");
14341        std::fs::create_dir(&repo).unwrap();
14342        init_repo(&repo);
14343        let mut runner = runner_at(RunStatus::Implementing);
14344        runner.state.repo = repo.clone();
14345        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14346        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14347        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14348            let branch = format!("candidate-{label}");
14349            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14350            git::worktree_add_branch(&repo, &wt, &branch, "main")
14351                .await
14352                .unwrap();
14353            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14354            git::git(&wt, &["add", "README.md"]).await.unwrap();
14355            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14356            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14357            runner.state.candidates.push(Candidate {
14358                index,
14359                label,
14360                agent: "alpha".to_owned(),
14361                branch,
14362                worktree: wt,
14363                summary: "completed implementation".to_owned(),
14364                stat: String::new(),
14365                files: 0,
14366                commits: 0,
14367                empty: false,
14368                failed: None,
14369                verified_noop: None,
14370                duration_ms: 42,
14371                folded: false,
14372            });
14373            runner
14374                .state
14375                .candidate_preservation
14376                .insert(label, "awaiting persistence".to_owned());
14377        }
14378        let wt = runner.state.candidates[0].worktree.clone();
14379        let index_lock = git::git(
14380            &wt,
14381            &[
14382                "rev-parse",
14383                "--path-format=absolute",
14384                "--git-path",
14385                "index.lock",
14386            ],
14387        )
14388        .await
14389        .unwrap();
14390        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14391        runner.state.save().unwrap();
14392        assert!(runner.persist_candidates().await.is_err());
14393        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14394        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14395        assert_eq!(
14396            runner.state.candidates[1].commits, 1,
14397            "other results are still collected"
14398        );
14399        assert!(
14400            !runner.state.candidates[0].empty,
14401            "failed rescue is not no change"
14402        );
14403        assert_eq!(
14404            runner.state.candidates[0].summary,
14405            "completed implementation"
14406        );
14407        assert!(runner.after_implement().is_err());
14408        let home = tmp.path().join("fold-home");
14409        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14410        assert!(wt.join("new-file").exists());
14411        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14412        assert!(recovery.join("staged.patch").exists());
14413        assert!(recovery.join("unstaged.patch").exists());
14414        assert_eq!(
14415            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14416            "untracked edits\n"
14417        );
14418        runner.state.begin_pass();
14419        runner.state.end_pass_parked(ParkCause::Upgrade {
14420            from: "1.0".into(),
14421            to: Some("1.1".into()),
14422        });
14423        runner.state.save().unwrap();
14424        runner.state = RunState::load(&runner.state.id).unwrap();
14425        assert!(matches!(
14426            runner.state.passes.last().unwrap().end,
14427            Some(crate::run::PassEnd::Parked {
14428                cause: ParkCause::Upgrade { .. }
14429            })
14430        ));
14431        std::fs::remove_file(index_lock).unwrap();
14432        // No implementer is configured: a persistence retry cannot dispatch a seat.
14433        runner.implement().await.unwrap();
14434        assert!(runner.state.candidate_preservation.is_empty());
14435        assert!(
14436            runner
14437                .state
14438                .candidates
14439                .iter()
14440                .all(|c| c.commits == 1 && !c.empty)
14441        );
14442        assert!(git::is_clean(&wt).await.unwrap());
14443        assert!(
14444            crate::run::read_artifact(&runner.state, "cand-A.patch")
14445                .unwrap()
14446                .contains("later tracked edits")
14447        );
14448    }
14449
14450    #[tokio::test]
14451    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14452        ask_test_home();
14453        let tmp = tempfile::tempdir().unwrap();
14454        let repo = tmp.path().join("repo");
14455        std::fs::create_dir(&repo).unwrap();
14456        init_repo(&repo);
14457        let wt = tmp.path().join("candidate");
14458        git::worktree_add_branch(&repo, &wt, "candidate", "main")
14459            .await
14460            .unwrap();
14461        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14462        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14463        assert!(wt.join("uncommitted").exists());
14464        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14465        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14466        let mut runner = runner_at(RunStatus::Implementing);
14467        runner.state.repo = repo.clone();
14468        runner.state.base_commit = "missing-base".to_owned();
14469        runner.state.candidates.push(Candidate {
14470            index: 0,
14471            label: 'A',
14472            agent: "alpha".into(),
14473            branch: "candidate".into(),
14474            worktree: wt.clone(),
14475            summary: "completed".into(),
14476            stat: String::new(),
14477            files: 0,
14478            commits: 0,
14479            empty: false,
14480            failed: None,
14481            verified_noop: None,
14482            duration_ms: 1,
14483            folded: false,
14484        });
14485        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14486        assert!(
14487            fold_run(&mut runner.state, true, &tmp.path().join("home"))
14488                .await
14489                .is_err()
14490        );
14491        assert!(!runner.state.candidates[0].folded);
14492        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14493        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14494        runner
14495            .state
14496            .candidate_preservation
14497            .insert('A', "pending".into());
14498        assert!(runner.persist_candidates().await.is_err());
14499        assert!(!runner.state.candidates[0].empty);
14500        assert!(!runner.state.candidates[0].folded);
14501        assert!(wt.exists());
14502        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14503        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14504        std::fs::create_dir_all(&patch_path).unwrap();
14505        assert!(
14506            runner.persist_candidates().await.is_err(),
14507            "artifact failure stays pending"
14508        );
14509        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14510        assert!(!runner.state.candidates[0].empty);
14511        std::fs::remove_dir(patch_path).unwrap();
14512        runner.persist_candidates().await.unwrap();
14513        assert!(
14514            runner.state.candidates[0].empty,
14515            "only successful inspection confirms no change"
14516        );
14517    }
14518
14519    fn init_repo(dir: &Path) {
14520        let run = |args: &[&str]| {
14521            let out = std::process::Command::new("git")
14522                .args(args)
14523                .current_dir(dir)
14524                .quiet()
14525                .output()
14526                .expect("spawn git");
14527            assert!(
14528                out.status.success(),
14529                "git {args:?} failed: {}",
14530                String::from_utf8_lossy(&out.stderr)
14531            );
14532        };
14533        run(&["init", "-b", "main"]);
14534        run(&["config", "user.name", "magi test"]);
14535        run(&["config", "user.email", "magi@example.com"]);
14536        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14537        run(&["add", "-A"]);
14538        run(&["commit", "-m", "init"]);
14539    }
14540
14541    // `settle_questions` is what closes the ghost the phone showed: a run's
14542    // seat asked something, the run then ended, and nothing was left to
14543    // abandon the question it left `open`. `HOME` is a process-wide
14544    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
14545    // first time it runs in the binary — every test below still reaches the
14546    // same directory whichever call won, and each gets its own run id from
14547    // `RunState::new`, so they never collide there.
14548    fn ask_test_home() {
14549        crate::run::pin_test_home();
14550    }
14551
14552    /// A minimal, git-free `Runner` at a given status — `settle_questions`
14553    /// reads nothing else off it.
14554    fn runner_at(status: RunStatus) -> Runner {
14555        let mut state = RunState::new(
14556            PathBuf::from("/nonexistent/repo"),
14557            "main".to_owned(),
14558            "deadbeef".to_owned(),
14559            "task".to_owned(),
14560            Config::default(),
14561        );
14562        // These fixtures share one home and save concurrently. Short clock
14563        // IDs can collide, making an ownership test read another test's run.
14564        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14565        state.id = format!(
14566            "20000101-000000-{:04x}",
14567            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14568        );
14569        state.status = status;
14570        Runner {
14571            state,
14572            roles: ResolvedRoles {
14573                implementers: Vec::new(),
14574                judges: Vec::new(),
14575                reviewers: Vec::new(),
14576                fixer: None,
14577                conductor: conductor(),
14578                implementer_roster: Vec::new(),
14579                judge_roster: Vec::new(),
14580                reviewer_roster: Vec::new(),
14581            },
14582            sem: Arc::new(Semaphore::new(1)),
14583            pause: Pause::new(),
14584            interrupt: Pause::new(),
14585        }
14586    }
14587
14588    #[tokio::test]
14589    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
14590        crate::run::pin_test_home();
14591        let mut runner = runner_at(RunStatus::Gating);
14592        runner.state.save().unwrap();
14593        let id = runner.state.id.clone();
14594        let passes = runner.state.passes.len();
14595        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
14596        let mut claiming = Box::pin(runner.claim_driver());
14597        std::future::poll_fn(|cx| {
14598            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
14599            std::task::Poll::Ready(())
14600        })
14601        .await;
14602        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
14603        drop(owner);
14604        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
14605            .await
14606            .unwrap()
14607            .unwrap();
14608        assert_eq!(execution.runner.state.passes.len(), passes + 1);
14609        assert!(execution.runner.state.resource_block.is_none());
14610        assert_eq!(
14611            execution.runner.state.liveness(false),
14612            crate::run::Liveness::Live
14613        );
14614        drop(execution);
14615        assert!(RunState::load(&id).unwrap().driver_exited);
14616    }
14617
14618    #[tokio::test]
14619    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14620        crate::run::pin_test_home();
14621        let mut runner = runner_at(RunStatus::Gating);
14622        runner.state.save().unwrap();
14623        // run.json can commit, but the subsequent lease metadata write fails.
14624        // Cleanup must run after releasing the short ownership lock.
14625        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14626        assert!(runner.claim_driver().await.is_err());
14627        assert!(runner.state.driver_exited);
14628        assert_eq!(runner.state.driver_pid, None);
14629        let disk = RunState::load(&runner.state.id).unwrap();
14630        assert!(disk.driver_exited);
14631        assert_eq!(disk.driver_pid, None);
14632        assert_eq!(disk.status, RunStatus::Gating);
14633    }
14634
14635    #[tokio::test]
14636    async fn an_early_graph_error_releases_its_driver() {
14637        crate::run::pin_test_home();
14638        let mut runner = runner_at(RunStatus::Blocked);
14639        runner.state.config.graph.selected_implementer = Some("selected".into());
14640        runner.state.save().unwrap();
14641        assert!(runner.execute().await.is_err());
14642        let disk = RunState::load(&runner.state.id).unwrap();
14643        assert!(disk.driver_exited);
14644        assert_eq!(disk.driver_pid, None);
14645        assert_eq!(disk.status, RunStatus::Blocked);
14646    }
14647
14648    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14649        crate::run::MergeRepair {
14650            question: "repair-question".into(),
14651            head: "old-head".into(),
14652            pr: "https://github.com/o/r/pull/42".into(),
14653            reviews: vec![review_round(false, 1, 1, 1, false, true)],
14654            gate: vec![outcome(Some(0), false)],
14655            gate_head: Some("old-head".into()),
14656            gate_fixes: Vec::new(),
14657            pre_gate: Vec::new(),
14658            phase,
14659            failure: None,
14660        }
14661    }
14662
14663    #[tokio::test]
14664    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14665        crate::run::pin_test_home();
14666        let mut runner = runner_at(RunStatus::Landing);
14667        runner.state.merge_repairs.push(merge_repair_request(
14668            crate::run::MergeRepairPhase::Requested,
14669        ));
14670        runner.state.save().unwrap();
14671        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14672        assert!(
14673            runner
14674                .execute()
14675                .await
14676                .unwrap_err()
14677                .to_string()
14678                .contains("already running")
14679        );
14680        assert_eq!(
14681            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14682            crate::run::MergeRepairPhase::Requested
14683        );
14684        drop(claim);
14685        let mut disk = runner.state.clone();
14686        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14687        disk.save().unwrap();
14688        assert!(
14689            runner
14690                .execute()
14691                .await
14692                .unwrap_err()
14693                .to_string()
14694                .contains("decision changed")
14695        );
14696        assert_eq!(
14697            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14698            crate::run::MergeRepairPhase::Complete
14699        );
14700    }
14701
14702    #[test]
14703    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14704        crate::run::pin_test_home();
14705        let mut runner = runner_at(RunStatus::Landing);
14706        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14707        runner.state.reviews = request.reviews.clone();
14708        runner.state.gate = request.gate.clone();
14709        runner.state.gate_ran = true;
14710        runner.state.land_approval = Some(crate::run::LandApproval {
14711            question: "old".into(),
14712            head: "old-head".into(),
14713        });
14714        runner.state.merge_repairs.push(request);
14715        runner.begin_merge_repair_review(0).unwrap();
14716        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14717        assert!(!runner.state.gate_status().ok());
14718        assert!(runner.state.land_approval.is_none());
14719        let disk = RunState::load(&runner.state.id).unwrap();
14720        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14721        assert!(disk.merge_repairs[0].gate[0].ok());
14722        assert_eq!(
14723            disk.merge_repairs[0].phase,
14724            crate::run::MergeRepairPhase::Reviewing
14725        );
14726        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14727        assert!(disk.reviews.is_empty());
14728    }
14729
14730    #[tokio::test]
14731    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14732        crate::run::pin_test_home();
14733        for phase in [
14734            crate::run::MergeRepairPhase::Fixing,
14735            crate::run::MergeRepairPhase::Requested,
14736        ] {
14737            let mut runner = runner_at(RunStatus::Landing);
14738            runner.state.config.graph.land_rounds = 0;
14739            runner.state.merge_repairs.push(merge_repair_request(phase));
14740            runner.state.save().unwrap();
14741            assert!(!runner.repair_merge_review().await.unwrap());
14742            assert_eq!(runner.state.status, RunStatus::Blocked);
14743            assert_eq!(
14744                runner.state.merge_repairs[0].phase,
14745                crate::run::MergeRepairPhase::Failed
14746            );
14747            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14748            assert!(
14749                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14750                    "interrupted"
14751                } else {
14752                    "budget exhausted"
14753                })
14754            );
14755            assert!(runner.state.merge.is_none());
14756        }
14757    }
14758
14759    /// `park_here` folding in the reason `Pause::park_because` recorded -
14760    /// this is what lets an operator reading a run's events tell an
14761    /// interrupt-driven park from an ordinary shutdown park.
14762    #[test]
14763    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14764        crate::run::pin_test_home();
14765        let mut runner = runner_at(RunStatus::Implementing);
14766        let interrupt = Pause::new();
14767        runner.watch_interrupt(interrupt.clone());
14768
14769        interrupt.park_because("task a1b2 asked to run first");
14770
14771        assert!(runner.park_here().expect("park_here"));
14772        assert!(runner.state.parked);
14773        let last = runner.state.events.last().expect("a park event");
14774        assert_eq!(last.node, "park");
14775        assert!(
14776            last.message.contains("task a1b2 asked to run first"),
14777            "expected the interrupt reason in {:?}",
14778            last.message
14779        );
14780    }
14781
14782    /// The cause is stored on the pass when it parks, and the first cause a
14783    /// shared pause was given is the one that stays.
14784    #[test]
14785    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14786        crate::run::pin_test_home();
14787        let mut runner = runner_at(RunStatus::Implementing);
14788        runner.state.begin_pass();
14789        let shutdown = Pause::new();
14790        runner.on_pause(shutdown.clone());
14791        let upgrade = ParkCause::Upgrade {
14792            from: "0.121.0".to_owned(),
14793            to: Some("0.121.1".to_owned()),
14794        };
14795        shutdown.park_for(upgrade.clone());
14796        shutdown.park_for(ParkCause::Operator);
14797
14798        assert!(runner.park_here().expect("park_here"));
14799        assert_eq!(
14800            runner.state.passes[0].end,
14801            Some(crate::run::PassEnd::Parked { cause: upgrade })
14802        );
14803        let last = runner.state.events.last().expect("a park event");
14804        assert!(last.message.contains("0.121.0"), "{}", last.message);
14805    }
14806
14807    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
14808    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
14809    /// cleared) must not make a *different* run - one only watching its own,
14810    /// unshared interrupt `Pause` - see itself as parked. If a future change
14811    /// ever collapsed these back into one handle, the interrupt scheduler
14812    /// would park every run for the rest of the daemon's life, not just the
14813    /// one it meant to interrupt.
14814    #[test]
14815    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14816        crate::run::pin_test_home();
14817        let mut runner = runner_at(RunStatus::Implementing);
14818        let shutdown = Pause::new();
14819        runner.on_pause(shutdown.clone());
14820        let interrupt = Pause::new();
14821        runner.watch_interrupt(interrupt.clone());
14822
14823        // Nobody has asked for anything yet.
14824        assert!(!runner.park_here().expect("park_here"));
14825        assert!(!runner.state.parked);
14826
14827        // Only the interrupt handle fires; the shutdown handle stays clear.
14828        interrupt.park_because("test");
14829        assert!(!shutdown.parked());
14830        assert!(runner.park_here().expect("park_here"));
14831    }
14832
14833    /// The property every prior attempt at this feature failed to pin down:
14834    /// asking a run to park while one of its nodes has a real, in-flight
14835    /// async operation running (an agent call, in production) must not cut
14836    /// that operation short. `park_here` is only ever consulted *between*
14837    /// `execute`'s node calls - see its own doc - so nothing inside a node
14838    /// can observe a park request until the node itself returns. This proves
14839    /// that structurally, with real `tokio` concurrency and a channel
14840    /// handshake (never a sleep, which would only prove "usually", not
14841    /// "cannot"): the "node" below reports that it has genuinely started,
14842    /// and only then is the park requested; the node still has to be told to
14843    /// finish before `park_here` is ever called, exactly mirroring every
14844    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
14845    /// in `execute`.
14846    #[tokio::test]
14847    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14848        crate::run::pin_test_home();
14849        let mut runner = runner_at(RunStatus::Implementing);
14850        let interrupt = Pause::new();
14851        runner.watch_interrupt(interrupt.clone());
14852
14853        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14854        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14855
14856        // Stands in for one node's in-flight agent call: it proves it has
14857        // genuinely started, then blocks - exactly as a spawned CLI process
14858        // does - until told to finish.
14859        let node = async move {
14860            started_tx.send(()).expect("send started");
14861            finish_rx.await.expect("recv finish");
14862            "node finished"
14863        };
14864
14865        let interrupter = async move {
14866            started_rx.await.expect("recv started");
14867            // The call is now genuinely in flight. Ask it to park.
14868            interrupt.park_because("higher-priority task waiting");
14869            // Nothing the node does can observe this yet - there is no
14870            // check inside it, by construction - so let the executor run
14871            // anything pending and then let the node finish on its own.
14872            tokio::task::yield_now().await;
14873            finish_tx.send(()).expect("send finish");
14874        };
14875
14876        let (node_result, ()) = tokio::join!(node, interrupter);
14877        assert_eq!(
14878            node_result, "node finished",
14879            "the in-flight call ran to completion"
14880        );
14881
14882        // Only now, at the boundary the real `execute` would check right
14883        // after this node, does the park take effect.
14884        assert!(runner.park_here().expect("park_here"));
14885        assert!(runner.state.parked);
14886    }
14887
14888    /// A run parked mid-competition carries every field it had accumulated
14889    /// through the exact same disk round-trip an ordinary resume uses -
14890    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
14891    /// Nothing about parking for an interrupt is a special case of that path;
14892    /// this is what proves it rather than assuming it.
14893    #[test]
14894    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14895        crate::run::pin_test_home();
14896        let mut runner = runner_at(RunStatus::Judging);
14897        // `Runner::resume` re-resolves roles from the saved config, which
14898        // refuses an empty roster - give it the same minimal one `conductor`
14899        // itself uses.
14900        runner.state.config.agents = vec![conductor()];
14901        runner.state.candidates = vec![Candidate {
14902            index: 0,
14903            label: 'A',
14904            agent: "alpha".to_owned(),
14905            branch: "magi/x/A".to_owned(),
14906            worktree: PathBuf::from("/nonexistent/worktree"),
14907            summary: "did the thing".to_owned(),
14908            stat: "1 file changed".to_owned(),
14909            files: 1,
14910            commits: 1,
14911            empty: false,
14912            failed: None,
14913            verified_noop: None,
14914            duration_ms: 1234,
14915            folded: false,
14916        }];
14917        let run_id = runner.state.id.clone();
14918
14919        let interrupt = Pause::new();
14920        runner.watch_interrupt(interrupt.clone());
14921        interrupt.park_because("task c3d4 asked to run first");
14922        assert!(runner.park_here().expect("park_here"));
14923
14924        let resumed = Runner::resume(&run_id).expect("resume");
14925        assert_eq!(resumed.state.candidates.len(), 1);
14926        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14927        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14928        assert_eq!(resumed.state.status, runner.state.status);
14929        assert!(
14930            resumed.state.parked,
14931            "still parked until `execute` actually walks the graph again"
14932        );
14933        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14934    }
14935
14936    /// A fresh open question on `run`, stored and handed back for assertions.
14937    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14938        let mut q = ask::Question::new(
14939            run.to_owned(),
14940            "implement".to_owned(),
14941            "impl-A".to_owned(),
14942            "Which storage backend should the cache use?".to_owned(),
14943            String::new(),
14944            vec!["SQLite".to_owned(), "Redis".to_owned()],
14945        );
14946        store.put(&mut q).unwrap();
14947        q
14948    }
14949
14950    #[test]
14951    fn a_failed_runs_open_question_is_abandoned() {
14952        ask_test_home();
14953        let store = ask::Questions::open();
14954        let mut runner = runner_at(RunStatus::Failed);
14955        let run = runner.state.id.clone();
14956        let q = ask_open_question(&store, &run);
14957
14958        runner.settle_questions();
14959
14960        let back = store.get(&q.id).unwrap();
14961        assert!(
14962            !back.status.open(),
14963            "the seat that asked died with the run; nobody is left to read an answer"
14964        );
14965        assert!(
14966            back.detail.contains(&run) && back.detail.contains("failed"),
14967            "the reason names what the run became, not just that it is gone: {}",
14968            back.detail
14969        );
14970    }
14971
14972    #[test]
14973    fn a_merged_runs_open_question_is_abandoned_too() {
14974        ask_test_home();
14975        let store = ask::Questions::open();
14976        // A run that finishes cleanly still leaves nobody to read an answer -
14977        // this is not only a failure-path cleanup.
14978        for status in [RunStatus::Merged, RunStatus::Ready] {
14979            let mut runner = runner_at(status);
14980            let run = runner.state.id.clone();
14981            let q = ask_open_question(&store, &run);
14982
14983            runner.settle_questions();
14984
14985            let back = store.get(&q.id).unwrap();
14986            assert!(
14987                !back.status.open(),
14988                "{status:?} run's question must not outlive the run"
14989            );
14990        }
14991    }
14992
14993    #[test]
14994    fn a_still_resumable_runs_open_question_is_left_alone() {
14995        ask_test_home();
14996        let store = ask::Questions::open();
14997        // `Blocked` and `Stalled` can still be resumed — the candidates, the
14998        // review round and the seat sessions are all still on disk — so a
14999        // question asked mid-round may yet get a real answer from a real
15000        // resume. Sweeping it here would be exactly the failure mode this
15001        // whole feature exists to avoid on the other side.
15002        for status in [RunStatus::Blocked, RunStatus::Stalled] {
15003            let mut runner = runner_at(status);
15004            let run = runner.state.id.clone();
15005            let q = ask_open_question(&store, &run);
15006
15007            runner.settle_questions();
15008
15009            let back = store.get(&q.id).unwrap();
15010            assert!(
15011                back.status.open(),
15012                "{status:?} is still alive; the question must still be waiting"
15013            );
15014        }
15015    }
15016
15017    #[test]
15018    fn settle_questions_never_touches_an_already_answered_question() {
15019        ask_test_home();
15020        let store = ask::Questions::open();
15021        let mut runner = runner_at(RunStatus::Failed);
15022        let run = runner.state.id.clone();
15023        let mut q = ask_open_question(&store, &run);
15024        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15025            .unwrap();
15026        store.put(&mut q).unwrap();
15027
15028        // Called twice, the way a crash-recovered daemon reclaim and the
15029        // graph's own cleanup both can for the same run — `abandon_for_run`
15030        // only ever touches what is still open, so this must be inert both
15031        // times, not merely the second.
15032        runner.settle_questions();
15033        runner.settle_questions();
15034
15035        let back = store.get(&q.id).unwrap();
15036        assert_eq!(
15037            back.status,
15038            ask::QuestionStatus::Answered,
15039            "a real answer is a decision on record, never overwritten by a sweep"
15040        );
15041    }
15042
15043    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
15044    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
15045    /// without merging, whose winner is still the operator's answer to read.
15046    /// Nothing previously called `fold_run` itself with a real `tally`, so
15047    /// this is the first test to pin down the one distinction the whole
15048    /// automatic-fold feature depends on: the winner's worktree and branch
15049    /// must survive, everything else sharing the run's worktree bay - a
15050    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
15051    /// second sweep treats every non-winner directory under the bay alike -
15052    /// must not.
15053    #[tokio::test]
15054    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15055        crate::run::pin_test_home();
15056        let tmp = tempfile::tempdir().expect("tempdir");
15057        let repo = tmp.path().join("repo");
15058        std::fs::create_dir_all(&repo).unwrap();
15059        init_repo(&repo);
15060
15061        let mut config = Config::default();
15062        config.graph.worktree_root = Some(tmp.path().join("wt"));
15063
15064        let mut state = RunState::new(
15065            repo.clone(),
15066            "main".to_owned(),
15067            "deadbeef".to_owned(),
15068            "task".to_owned(),
15069            config,
15070        );
15071        let root = state.worktree_root();
15072        let wt_a = root.join("cand-A");
15073        let wt_b = root.join("cand-B");
15074        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15075            .await
15076            .expect("worktree A");
15077        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15078            .await
15079            .expect("worktree B");
15080
15081        state.candidates = vec![
15082            Candidate {
15083                index: 0,
15084                label: 'A',
15085                agent: "alpha".to_owned(),
15086                branch: "magi/x/A".to_owned(),
15087                worktree: wt_a.clone(),
15088                summary: String::new(),
15089                stat: String::new(),
15090                files: 0,
15091                commits: 0,
15092                empty: false,
15093                failed: None,
15094                verified_noop: None,
15095                duration_ms: 0,
15096                folded: false,
15097            },
15098            Candidate {
15099                index: 1,
15100                label: 'B',
15101                agent: "beta".to_owned(),
15102                branch: "magi/x/B".to_owned(),
15103                worktree: wt_b.clone(),
15104                summary: String::new(),
15105                stat: String::new(),
15106                files: 0,
15107                commits: 0,
15108                empty: false,
15109                failed: None,
15110                verified_noop: None,
15111                duration_ms: 0,
15112                folded: false,
15113            },
15114        ];
15115        state.tally = Some(Tally {
15116            first_choice: BTreeMap::from([('A', 1)]),
15117            borda: BTreeMap::new(),
15118            winner: 'A',
15119            rankings: 1,
15120            unanimous_initial: true,
15121            deliberated: false,
15122            changed_votes: 0,
15123            unanimous_final: true,
15124            tie_break: None,
15125            judges: 1,
15126            present: 1,
15127            quorum: 1,
15128            met_quorum: true,
15129            uncontested: None,
15130        });
15131        state.status = RunStatus::Ready;
15132
15133        fold_run(&mut state, false, &crate::run::home())
15134            .await
15135            .expect("fold_run");
15136
15137        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15138        assert!(
15139            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15140            "the unmerged winner's branch survives"
15141        );
15142        assert!(
15143            !state.candidates[0].folded,
15144            "the winner is not marked folded"
15145        );
15146
15147        assert!(!wt_b.exists(), "the loser's worktree is removed");
15148        assert!(
15149            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15150            "the loser's branch is removed"
15151        );
15152        assert!(state.candidates[1].folded, "the loser is marked folded");
15153    }
15154
15155    #[tokio::test]
15156    #[cfg(unix)]
15157    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15158        use std::os::fd::AsRawFd as _;
15159        use std::time::Duration;
15160
15161        let tmp = tempfile::tempdir().unwrap();
15162        let repo = tmp.path().join("repo");
15163        std::fs::create_dir_all(&repo).unwrap();
15164        init_repo(&repo);
15165        let common = repo.join(".git");
15166        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15167        let home = tmp.path().join("home");
15168        let mut config = Config::default();
15169        config.coordination = crate::config::test_coordination();
15170        config.graph.worktree_root = Some(tmp.path().join("wt"));
15171        config.graph.timeout_resource_wait = 1;
15172        let mut state = RunState::new(
15173            repo.clone(),
15174            "main".into(),
15175            "deadbeef".into(),
15176            "task".into(),
15177            config,
15178        );
15179        let root = state.worktree_root();
15180        let candidate = root.join("cand-A");
15181        let reviewer = root.join("review-1");
15182        let branch = state.branch_for('A');
15183        git::worktree_add_branch(&repo, &candidate, &branch, "main")
15184            .await
15185            .unwrap();
15186        git::worktree_add_detached(&repo, &reviewer, "main")
15187            .await
15188            .unwrap();
15189        git::acquire_worktree_config(&repo).await.unwrap();
15190        state.enabled_worktree_config = true;
15191        state.status = RunStatus::Ready;
15192        state.candidates.push(Candidate {
15193            index: 0,
15194            label: 'A',
15195            agent: "implement".into(),
15196            branch: branch.clone(),
15197            worktree: candidate.clone(),
15198            summary: String::new(),
15199            stat: String::new(),
15200            files: 0,
15201            commits: 0,
15202            empty: false,
15203            failed: None,
15204            verified_noop: None,
15205            duration_ms: 0,
15206            folded: false,
15207        });
15208        state.save_under(&home).unwrap();
15209        let config_before = std::fs::read(common.join("config")).unwrap();
15210        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15211            .await
15212            .unwrap();
15213        // A separate OS handle bypasses the launcher's local gate, exactly as
15214        // an external updater does, and must still prevent all folding work.
15215        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15216        assert_eq!(
15217            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15218            0
15219        );
15220        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15221        assert!(
15222            tokio::time::timeout(Duration::from_millis(200), &mut fold)
15223                .await
15224                .is_err()
15225        );
15226        assert!(candidate.exists());
15227        assert!(reviewer.exists());
15228        assert!(git::branch_exists(&repo, &branch).await.unwrap());
15229        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15230        assert_eq!(
15231            git::git(&repo, &["worktree", "list", "--porcelain"])
15232                .await
15233                .unwrap(),
15234            worktrees_before
15235        );
15236        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15237            .await
15238            .unwrap()
15239            .unwrap_err();
15240        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15241        drop(fold);
15242        assert!(candidate.exists() && reviewer.exists());
15243        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15244        assert_eq!(
15245            state.resource_block.as_ref().unwrap().kind,
15246            "repository coordination"
15247        );
15248        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15249        drop(external);
15250        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15251        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15252            .await
15253            .unwrap()
15254            .unwrap();
15255        drop(fold);
15256        assert!(removed.contains(&branch));
15257        assert!(!root.exists());
15258        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15259        assert!(state.candidates[0].folded);
15260        assert!(!state.enabled_worktree_config);
15261        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15262        let enabled = git::git_raw(
15263            &repo,
15264            &["config", "--local", "--get", "extensions.worktreeConfig"],
15265        )
15266        .await
15267        .unwrap();
15268        assert!(!enabled.ok(), "the last config reference was released");
15269    }
15270
15271    /// A branch handed to a later run is that run's (and its pull request's):
15272    /// folding the run that released it must not delete it.
15273    #[tokio::test]
15274    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15275        let tmp = tempfile::tempdir().expect("tempdir");
15276        let repo = tmp.path().join("repo");
15277        std::fs::create_dir_all(&repo).unwrap();
15278        init_repo(&repo);
15279        let home = tmp.path().join("home");
15280
15281        let mut config = Config::default();
15282        config.graph.worktree_root = Some(tmp.path().join("wt"));
15283        let mut state = RunState::new(
15284            repo.clone(),
15285            "main".to_owned(),
15286            "deadbeef".to_owned(),
15287            "task".to_owned(),
15288            config,
15289        );
15290        // The worktree is already gone (released); the branch survives.
15291        git::git(&repo, &["branch", "magi/x/A", "main"])
15292            .await
15293            .expect("branch");
15294        state.candidates = vec![Candidate {
15295            index: 0,
15296            label: 'A',
15297            agent: "alpha".to_owned(),
15298            branch: "magi/x/A".to_owned(),
15299            worktree: state.worktree_root().join("cand-A"),
15300            summary: String::new(),
15301            stat: String::new(),
15302            files: 0,
15303            commits: 0,
15304            empty: false,
15305            failed: None,
15306            verified_noop: None,
15307            duration_ms: 0,
15308            folded: true,
15309        }];
15310        state.released_to = Some("20260901-000000-new1".to_owned());
15311        state.released_branches = vec!["magi/x/A".to_owned()];
15312
15313        fold_run(&mut state, true, &home).await.expect("fold_run");
15314        state.status = RunStatus::Merged;
15315        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15316        state.merge = Some(crate::run::MergeOutcome {
15317            mode: MergeMode::Pr,
15318            ok: true,
15319            detail: "confirmed merged".into(),
15320            empty: false,
15321        });
15322        fold_run(&mut state, true, &home)
15323            .await
15324            .expect("merged fold");
15325
15326        assert!(
15327            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15328            "the handed-over branch survives a fold"
15329        );
15330    }
15331
15332    /// A winner with nothing ahead of the base is caught before `gh` is ever
15333    /// asked for a pull request, and the message carries what the task's
15334    /// references resolved to.
15335    #[tokio::test]
15336    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15337        let tmp = tempfile::tempdir().expect("tempdir");
15338        let repo = tmp.path().join("repo");
15339        std::fs::create_dir_all(&repo).unwrap();
15340        init_repo(&repo);
15341        let run = |args: &[&str]| {
15342            let out = std::process::Command::new("git")
15343                .quiet()
15344                .args(args)
15345                .current_dir(&repo)
15346                .output()
15347                .expect("spawn git");
15348            assert!(out.status.success(), "git {args:?}");
15349        };
15350        run(&["branch", "magi/x/A"]);
15351        run(&["checkout", "-q", "-b", "magi/x/B"]);
15352        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15353        run(&["add", "-A"]);
15354        run(&["commit", "-q", "-m", "work"]);
15355        run(&["checkout", "-q", "main"]);
15356
15357        // A pull request is compared against the remote's base, so the
15358        // fixture needs one. Before it exists nothing can be read, and the
15359        // answer must be "not empty".
15360        let probe = RunState::new(
15361            repo.clone(),
15362            "main".to_owned(),
15363            "deadbeef".to_owned(),
15364            "task".to_owned(),
15365            Config::default(),
15366        );
15367        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15368        let bare = tmp.path().join("origin.git");
15369        let out = std::process::Command::new("git")
15370            .quiet()
15371            .args(["init", "-q", "--bare"])
15372            .arg(&bare)
15373            .output()
15374            .expect("spawn git");
15375        assert!(out.status.success(), "git init --bare");
15376        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15377        run(&["push", "-q", "origin", "main"]);
15378
15379        let mut state = RunState::new(
15380            repo.clone(),
15381            "main".to_owned(),
15382            "deadbeef".to_owned(),
15383            "task".to_owned(),
15384            Config::default(),
15385        );
15386        state.seeds = vec![refs::Seed {
15387            token: "magi/27b2/A".to_owned(),
15388            kind: refs::SeedKind::Unresolved,
15389            sha: String::new(),
15390            branch: true,
15391            detail: "no branch or commit named magi/27b2/A".to_owned(),
15392        }];
15393
15394        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15395        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15396        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15397        let detail = empty_candidate_detail(&state, "main");
15398        assert!(detail.starts_with("empty candidate"), "{detail}");
15399        assert!(!detail.contains("magi/27b2/A"), "{detail}");
15400        assert!(detail.contains("--start-from"), "{detail}");
15401    }
15402
15403    /// `status == Ready` used to be read as "this is the harmless
15404    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
15405    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
15406    /// run's PR was closed without merging — and reentering `merge` with
15407    /// `mode` still `Pr` does not know the difference, so it pushed and
15408    /// opened a second pull request. `mode == Local` reproduces the same
15409    /// blind spot without a network call: reentry must not attempt another
15410    /// git merge once this node has already recorded an outcome.
15411    #[tokio::test]
15412    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15413        let tmp = tempfile::tempdir().expect("tempdir");
15414        let repo = tmp.path().join("repo");
15415        std::fs::create_dir_all(&repo).unwrap();
15416        init_repo(&repo);
15417
15418        let mut config = Config::default();
15419        config.merge.mode = MergeMode::Local;
15420
15421        let mut state = RunState::new(
15422            repo.clone(),
15423            "main".to_owned(),
15424            "deadbeef".to_owned(),
15425            "task".to_owned(),
15426            config,
15427        );
15428        state.candidates = vec![Candidate {
15429            index: 0,
15430            label: 'A',
15431            agent: "alpha".to_owned(),
15432            branch: "does-not-exist".to_owned(),
15433            worktree: repo.clone(),
15434            summary: String::new(),
15435            stat: String::new(),
15436            files: 0,
15437            commits: 0,
15438            empty: false,
15439            failed: None,
15440            verified_noop: None,
15441            duration_ms: 0,
15442            folded: false,
15443        }];
15444        state.tally = Some(Tally {
15445            first_choice: BTreeMap::from([('A', 1)]),
15446            borda: BTreeMap::new(),
15447            winner: 'A',
15448            rankings: 1,
15449            unanimous_initial: true,
15450            deliberated: false,
15451            changed_votes: 0,
15452            unanimous_final: true,
15453            tie_break: None,
15454            judges: 0,
15455            present: 0,
15456            quorum: 0,
15457            met_quorum: true,
15458            uncontested: Some("only candidate A produced a change".to_owned()),
15459        });
15460        state.reviews = vec![ReviewRound {
15461            round: 1,
15462            head: "deadbeef".to_owned(),
15463            verified_head: None,
15464            verified_at: None,
15465            reviews: Vec::new(),
15466            e2e: Vec::new(),
15467            fix: None,
15468            blocking: 0,
15469            answered: 0,
15470            expected: 0,
15471            clean: true,
15472            verify_retried: false,
15473            e2e_deferred: false,
15474            e2e_defer_reason: None,
15475            progressed: false,
15476            vote_split: false,
15477            reconsideration: Vec::new(),
15478            verdict: None,
15479        }];
15480        state.gate = vec![CommandOutcome {
15481            command: "test".to_owned(),
15482            code: Some(0),
15483            output_tail: String::new(),
15484            duration_ms: 0,
15485            resource_blocked: false,
15486        }];
15487        state.gate_ran = true;
15488        // Reached its conclusion already — e.g. `land` closing the PR without
15489        // merging it, which (like the honest `MergeMode::None` path) leaves
15490        // `status` at `Ready`. The recorded outcome is what actually marks
15491        // this node done.
15492        state.status = RunStatus::Ready;
15493        state.merge = Some(MergeOutcome {
15494            mode: MergeMode::Local,
15495            ok: false,
15496            detail: "already concluded".to_owned(),
15497            empty: false,
15498        });
15499
15500        let mut runner = Runner {
15501            state,
15502            roles: ResolvedRoles {
15503                implementers: Vec::new(),
15504                judges: Vec::new(),
15505                reviewers: Vec::new(),
15506                fixer: None,
15507                conductor: conductor(),
15508                implementer_roster: Vec::new(),
15509                judge_roster: Vec::new(),
15510                reviewer_roster: Vec::new(),
15511            },
15512            sem: Arc::new(Semaphore::new(1)),
15513            pause: Pause::new(),
15514            interrupt: Pause::new(),
15515        };
15516
15517        runner.merge().await.expect("merge");
15518
15519        assert_eq!(
15520            runner.state.status,
15521            RunStatus::Ready,
15522            "a concluded run's status must not change on reentry"
15523        );
15524        assert_eq!(
15525            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15526            Some("already concluded"),
15527            "merge must not run again once the node already recorded an outcome"
15528        );
15529    }
15530
15531    /// `gate` leaves `state.gate_ran` false both before it has ever run and
15532    /// when its last attempt was resource-blocked (the shared build cache
15533    /// could not be acquired or confirmed fresh in time - see
15534    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
15535    /// `Vec` this also leaves behind used to read as "nothing failed" and let
15536    /// a run merge a tree the gate never actually checked - exactly the case
15537    /// a contended cache produces on every retry until it clears. `merge`
15538    /// must refuse until `gate` has actually recorded an attempt.
15539    #[tokio::test]
15540    async fn merge_refuses_a_gate_that_has_not_actually_run() {
15541        let tmp = tempfile::tempdir().expect("tempdir");
15542        let repo = tmp.path().join("repo");
15543        std::fs::create_dir_all(&repo).unwrap();
15544        init_repo(&repo);
15545
15546        let mut config = Config::default();
15547        config.merge.mode = MergeMode::Local;
15548
15549        let mut state = RunState::new(
15550            repo.clone(),
15551            "main".to_owned(),
15552            "deadbeef".to_owned(),
15553            "task".to_owned(),
15554            config,
15555        );
15556        state.candidates = vec![Candidate {
15557            index: 0,
15558            label: 'A',
15559            agent: "alpha".to_owned(),
15560            branch: "does-not-exist".to_owned(),
15561            worktree: repo.clone(),
15562            summary: String::new(),
15563            stat: String::new(),
15564            files: 0,
15565            commits: 0,
15566            empty: false,
15567            failed: None,
15568            verified_noop: None,
15569            duration_ms: 0,
15570            folded: false,
15571        }];
15572        state.tally = Some(Tally {
15573            first_choice: BTreeMap::from([('A', 1)]),
15574            borda: BTreeMap::new(),
15575            winner: 'A',
15576            rankings: 1,
15577            unanimous_initial: true,
15578            deliberated: false,
15579            changed_votes: 0,
15580            unanimous_final: true,
15581            tie_break: None,
15582            judges: 0,
15583            present: 0,
15584            quorum: 0,
15585            met_quorum: true,
15586            uncontested: Some("only candidate A produced a change".to_owned()),
15587        });
15588        state.reviews = vec![ReviewRound {
15589            round: 1,
15590            head: "deadbeef".to_owned(),
15591            verified_head: None,
15592            verified_at: None,
15593            reviews: Vec::new(),
15594            e2e: Vec::new(),
15595            fix: None,
15596            blocking: 0,
15597            answered: 0,
15598            expected: 0,
15599            clean: true,
15600            verify_retried: false,
15601            e2e_deferred: false,
15602            e2e_defer_reason: None,
15603            progressed: false,
15604            vote_split: false,
15605            reconsideration: Vec::new(),
15606            verdict: None,
15607        }];
15608        // The point: `gate` has not recorded anything yet.
15609        state.gate = Vec::new();
15610        state.gate_ran = false;
15611        state.status = RunStatus::Gating;
15612
15613        let mut runner = Runner {
15614            state,
15615            roles: ResolvedRoles {
15616                implementers: Vec::new(),
15617                judges: Vec::new(),
15618                reviewers: Vec::new(),
15619                fixer: None,
15620                conductor: conductor(),
15621                implementer_roster: Vec::new(),
15622                judge_roster: Vec::new(),
15623                reviewer_roster: Vec::new(),
15624            },
15625            sem: Arc::new(Semaphore::new(1)),
15626            pause: Pause::new(),
15627            interrupt: Pause::new(),
15628        };
15629
15630        runner.merge().await.expect("merge");
15631
15632        assert!(
15633            runner.state.merge.is_none(),
15634            "an empty gate must never be read as a passing one: {:?}",
15635            runner.state.merge
15636        );
15637    }
15638
15639    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
15640    /// commands configured and `merge.mode` is `none` (a review-only run).
15641    /// `gate` must still record a real attempt — zero commands, vacuously
15642    /// passed — rather than leaving `state.gate` empty in a way `merge`
15643    /// cannot tell apart from "never ran"; otherwise the run reaches
15644    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
15645    #[tokio::test]
15646    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15647        let tmp = tempfile::tempdir().expect("tempdir");
15648        let repo = tmp.path().join("repo");
15649        std::fs::create_dir_all(&repo).unwrap();
15650        init_repo(&repo);
15651
15652        // Default config: `verify.gate` empty, `merge.mode` is `none`.
15653        let config = Config::default();
15654
15655        let mut state = RunState::new(
15656            repo.clone(),
15657            "main".to_owned(),
15658            "deadbeef".to_owned(),
15659            "task".to_owned(),
15660            config,
15661        );
15662        state.candidates = vec![Candidate {
15663            index: 0,
15664            label: 'A',
15665            agent: "alpha".to_owned(),
15666            branch: "does-not-exist".to_owned(),
15667            worktree: repo.clone(),
15668            summary: String::new(),
15669            stat: String::new(),
15670            files: 0,
15671            commits: 0,
15672            empty: false,
15673            failed: None,
15674            verified_noop: None,
15675            duration_ms: 0,
15676            folded: false,
15677        }];
15678        state.tally = Some(Tally {
15679            first_choice: BTreeMap::from([('A', 1)]),
15680            borda: BTreeMap::new(),
15681            winner: 'A',
15682            rankings: 1,
15683            unanimous_initial: true,
15684            deliberated: false,
15685            changed_votes: 0,
15686            unanimous_final: true,
15687            tie_break: None,
15688            judges: 0,
15689            present: 0,
15690            quorum: 0,
15691            met_quorum: true,
15692            uncontested: Some("only candidate A produced a change".to_owned()),
15693        });
15694        state.reviews = vec![ReviewRound {
15695            round: 1,
15696            head: "deadbeef".to_owned(),
15697            verified_head: None,
15698            verified_at: None,
15699            reviews: Vec::new(),
15700            e2e: Vec::new(),
15701            fix: None,
15702            blocking: 0,
15703            answered: 0,
15704            expected: 0,
15705            clean: true,
15706            verify_retried: false,
15707            e2e_deferred: false,
15708            e2e_defer_reason: None,
15709            progressed: false,
15710            vote_split: false,
15711            reconsideration: Vec::new(),
15712            verdict: None,
15713        }];
15714
15715        let mut runner = Runner {
15716            state,
15717            roles: ResolvedRoles {
15718                implementers: Vec::new(),
15719                judges: Vec::new(),
15720                reviewers: Vec::new(),
15721                fixer: None,
15722                conductor: conductor(),
15723                implementer_roster: Vec::new(),
15724                judge_roster: Vec::new(),
15725                reviewer_roster: Vec::new(),
15726            },
15727            sem: Arc::new(Semaphore::new(1)),
15728            pause: Pause::new(),
15729            interrupt: Pause::new(),
15730        };
15731
15732        runner.gate().await.expect("gate");
15733        assert!(
15734            runner.state.gate_ran,
15735            "zero configured commands is still a real attempt, not an unrun gate"
15736        );
15737        assert!(runner.state.gate.is_empty());
15738        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15739        assert_ne!(
15740            runner.state.status,
15741            RunStatus::Blocked,
15742            "a gate with nothing to check must not read as failed"
15743        );
15744
15745        runner.merge().await.expect("merge");
15746        assert_eq!(
15747            runner.state.status,
15748            RunStatus::Ready,
15749            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15750        );
15751    }
15752
15753    /// `Config::cache_dir` is derived from `verify.e2e` as well as
15754    /// `verify.gate` (so the e2e leg and the final gate never build against
15755    /// different directories). With zero `verify.gate` commands but a
15756    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
15757    /// that lease before discovering it had nothing to run - so a repo with
15758    /// no gate commands could come back `resource_blocked` (and therefore
15759    /// still `gate_ran == false`) on nothing but an unrelated run holding the
15760    /// cache, exactly the contention this run's own zero commands could
15761    /// never have touched. `gate` must recognise there is nothing to check
15762    /// before it ever asks for the lease.
15763    #[tokio::test]
15764    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15765        crate::run::pin_test_home();
15766        let home = crate::run::home();
15767
15768        let tmp = tempfile::tempdir().expect("tempdir");
15769        let repo = tmp.path().join("repo");
15770        std::fs::create_dir_all(&repo).unwrap();
15771        init_repo(&repo);
15772        // Unique to this test, so holding its lease cannot collide with
15773        // another test sharing the same process-wide `home`.
15774        let cache_dir = tmp.path().join("target");
15775
15776        let mut config = Config::default();
15777        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15778        // `verify.gate` stays empty (the default). Bounded so a regression
15779        // that does start waiting fails the test in seconds, not hangs it.
15780        config.graph.timeout_verify = Some(2);
15781        config.graph.timeout_resource_wait = 2;
15782
15783        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15784        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15785            .expect("no io error acquiring directly")
15786        {
15787            crate::cache::AcquireOutcome::Acquired(g) => g,
15788            crate::cache::AcquireOutcome::Busy(b) => {
15789                panic!("expected the direct acquire to win the lease first: {b:?}")
15790            }
15791        };
15792
15793        let mut state = RunState::new(
15794            repo.clone(),
15795            "main".to_owned(),
15796            "deadbeef".to_owned(),
15797            "task".to_owned(),
15798            config,
15799        );
15800        state.candidates = vec![Candidate {
15801            index: 0,
15802            label: 'A',
15803            agent: "alpha".to_owned(),
15804            branch: "does-not-exist".to_owned(),
15805            worktree: repo.clone(),
15806            summary: String::new(),
15807            stat: String::new(),
15808            files: 0,
15809            commits: 0,
15810            empty: false,
15811            failed: None,
15812            verified_noop: None,
15813            duration_ms: 0,
15814            folded: false,
15815        }];
15816        state.tally = Some(Tally {
15817            first_choice: BTreeMap::from([('A', 1)]),
15818            borda: BTreeMap::new(),
15819            winner: 'A',
15820            rankings: 1,
15821            unanimous_initial: true,
15822            deliberated: false,
15823            changed_votes: 0,
15824            unanimous_final: true,
15825            tie_break: None,
15826            judges: 0,
15827            present: 0,
15828            quorum: 0,
15829            met_quorum: true,
15830            uncontested: Some("only candidate A produced a change".to_owned()),
15831        });
15832        state.reviews = vec![ReviewRound {
15833            round: 1,
15834            head: "deadbeef".to_owned(),
15835            verified_head: None,
15836            verified_at: None,
15837            reviews: Vec::new(),
15838            e2e: Vec::new(),
15839            fix: None,
15840            blocking: 0,
15841            answered: 0,
15842            expected: 0,
15843            clean: true,
15844            verify_retried: false,
15845            e2e_deferred: false,
15846            e2e_defer_reason: None,
15847            progressed: false,
15848            vote_split: false,
15849            reconsideration: Vec::new(),
15850            verdict: None,
15851        }];
15852
15853        let mut runner = Runner {
15854            state,
15855            roles: ResolvedRoles {
15856                implementers: Vec::new(),
15857                judges: Vec::new(),
15858                reviewers: Vec::new(),
15859                fixer: None,
15860                conductor: conductor(),
15861                implementer_roster: Vec::new(),
15862                judge_roster: Vec::new(),
15863                reviewer_roster: Vec::new(),
15864            },
15865            sem: Arc::new(Semaphore::new(1)),
15866            pause: Pause::new(),
15867            interrupt: Pause::new(),
15868        };
15869
15870        let started = std::time::Instant::now();
15871        runner.gate().await.expect("gate");
15872        assert!(
15873            started.elapsed() < Duration::from_secs(1),
15874            "a gate with nothing to run must never wait on a lease it never needed"
15875        );
15876        assert!(
15877            runner.state.gate_ran,
15878            "zero commands is still a real, immediate attempt"
15879        );
15880        assert!(runner.state.gate.is_empty());
15881        assert_ne!(
15882            runner.state.status,
15883            RunStatus::Blocked,
15884            "must not read as resource-blocked on a lease it never asked for"
15885        );
15886    }
15887
15888    #[cfg(unix)]
15889    #[tokio::test]
15890    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15891        crate::run::pin_test_home();
15892        let tmp = tempfile::tempdir().unwrap();
15893        let mut config = Config::default();
15894        config.disk.min_free_bytes = 0;
15895        let mut state = RunState::new(
15896            tmp.path().to_owned(),
15897            "main".into(),
15898            "base".into(),
15899            "verification evidence".into(),
15900            config,
15901        );
15902        let (outcomes, pids) = run_commands(
15903            &mut state,
15904            "gate",
15905            "gate",
15906            1,
15907            &["sh".into(), "-c".into()],
15908            &[
15909                "printf before-timeout; exec sleep 30".into(),
15910                "kill -TERM $$".into(),
15911                "exit 7".into(),
15912                "printf success".into(),
15913            ],
15914            tmp.path(),
15915            Duration::from_secs(1),
15916        )
15917        .await;
15918        wait_for_timed_out_children_to_die(&pids).await;
15919        assert_eq!(
15920            pids.len(),
15921            1,
15922            "unleased timeout retains its death-check PID"
15923        );
15924        assert_eq!(outcomes.len(), 4);
15925        assert_eq!(outcomes[0].code, None);
15926        assert!(outcomes[0].output_tail.contains("before-timeout"));
15927        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15928        assert_eq!(outcomes[1].code, None);
15929        assert!(outcomes[1].output_tail.contains("command terminated:"));
15930        assert!(!outcomes[1].output_tail.contains("timed out"));
15931        assert_eq!(outcomes[2].code, Some(7));
15932        assert!(!outcomes[2].output_tail.contains("timed out"));
15933        assert_eq!(outcomes[3].code, Some(0));
15934        assert_eq!(outcomes[3].output_tail, "success");
15935        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15936    }
15937
15938    /// The addendum's second gap: a `verify.gate` command running for real
15939    /// wall-clock time had nothing at all to show for it in `active` before
15940    /// `run_commands` learned to record it — a run could sit in `Gating` for
15941    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
15942    /// what was actually happening. Proven with a genuinely still-running
15943    /// command, not just a before/after check on the final state: a poller
15944    /// task reads the same `run.json` `gate()` is writing, the same way the
15945    /// phone or `magi show` would, while the shell command is still blocked
15946    /// on its own release marker.
15947    #[tokio::test]
15948    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15949        crate::run::pin_test_home();
15950
15951        let tmp = tempfile::tempdir().expect("tempdir");
15952        let repo = tmp.path().join("repo");
15953        std::fs::create_dir_all(&repo).unwrap();
15954        init_repo(&repo);
15955
15956        let mut config = Config::default();
15957        config.verify.gate = vec![
15958            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15959             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15960                .to_owned(),
15961        ];
15962
15963        let mut state = RunState::new(
15964            repo.clone(),
15965            "main".to_owned(),
15966            "deadbeef".to_owned(),
15967            "task".to_owned(),
15968            config,
15969        );
15970        let run_id = state.id.clone();
15971        state.candidates = vec![Candidate {
15972            index: 0,
15973            label: 'A',
15974            agent: "alpha".to_owned(),
15975            branch: "does-not-exist".to_owned(),
15976            worktree: repo.clone(),
15977            summary: String::new(),
15978            stat: String::new(),
15979            files: 0,
15980            commits: 0,
15981            empty: false,
15982            failed: None,
15983            verified_noop: None,
15984            duration_ms: 0,
15985            folded: false,
15986        }];
15987        state.tally = Some(Tally {
15988            first_choice: BTreeMap::from([('A', 1)]),
15989            borda: BTreeMap::new(),
15990            winner: 'A',
15991            rankings: 1,
15992            unanimous_initial: true,
15993            deliberated: false,
15994            changed_votes: 0,
15995            unanimous_final: true,
15996            tie_break: None,
15997            judges: 0,
15998            present: 0,
15999            quorum: 0,
16000            met_quorum: true,
16001            uncontested: Some("only candidate A produced a change".to_owned()),
16002        });
16003        state.reviews = vec![ReviewRound {
16004            round: 1,
16005            head: "deadbeef".to_owned(),
16006            verified_head: None,
16007            verified_at: None,
16008            reviews: Vec::new(),
16009            e2e: Vec::new(),
16010            fix: None,
16011            blocking: 0,
16012            answered: 0,
16013            expected: 0,
16014            clean: true,
16015            verify_retried: false,
16016            e2e_deferred: false,
16017            e2e_defer_reason: None,
16018            progressed: false,
16019            vote_split: false,
16020            reconsideration: Vec::new(),
16021            verdict: None,
16022        }];
16023
16024        let mut runner = Runner {
16025            state,
16026            roles: ResolvedRoles {
16027                implementers: Vec::new(),
16028                judges: Vec::new(),
16029                reviewers: Vec::new(),
16030                fixer: None,
16031                conductor: conductor(),
16032                implementer_roster: Vec::new(),
16033                judge_roster: Vec::new(),
16034                reviewer_roster: Vec::new(),
16035            },
16036            sem: Arc::new(Semaphore::new(1)),
16037            pause: Pause::new(),
16038            interrupt: Pause::new(),
16039        };
16040
16041        let started_marker = repo.join("started.marker");
16042        let release_marker = repo.join("release.marker");
16043        let poller = tokio::spawn(async move {
16044            // Bounded so a regression that never records the task entry
16045            // fails this test in seconds instead of hanging the suite —
16046            // the same shape `a_park_requested_while_a_seat_is_mid_call_
16047            // does_not_cut_it_short` uses for the same reason.
16048            for _ in 0..100 {
16049                if started_marker.exists()
16050                    && let Ok(s) = crate::run::RunState::load(&run_id)
16051                    && let Some(a) = s.active.get("gate")
16052                {
16053                    std::fs::write(&release_marker, b"go").expect("release marker");
16054                    return Some(a.clone());
16055                }
16056                tokio::time::sleep(Duration::from_millis(50)).await;
16057            }
16058            None
16059        });
16060
16061        runner.gate().await.expect("gate");
16062        let captured = poller.await.expect("poller task");
16063        let captured = captured.expect(
16064            "the poller never saw a `gate` task entry in run.json while the command was \
16065             still blocked on its own release marker",
16066        );
16067
16068        assert_eq!(captured.task.as_deref(), Some("gate"));
16069        assert_eq!(captured.node, "gate");
16070        assert_eq!(captured.index, Some(1));
16071        assert_eq!(captured.total, Some(1));
16072        assert!(
16073            captured
16074                .command
16075                .as_deref()
16076                .is_some_and(|c| c.contains("started.marker")),
16077            "{captured:?}"
16078        );
16079
16080        assert!(
16081            runner.state.active.is_empty(),
16082            "the entry must be cleared once the command actually finished: {:?}",
16083            runner.state.active
16084        );
16085        assert!(runner.state.gate_ran);
16086        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16087    }
16088
16089    /// The hand-off over a blocking finding a reviewer rejected on leaves a
16090    /// record for `land`; one with only a Minor, or no reject, leaves none.
16091    #[tokio::test]
16092    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16093        use crate::verdict::{Finding, ReviewVote, Severity};
16094        crate::run::pin_test_home();
16095        let tmp = tempfile::tempdir().expect("tempdir");
16096        let repo = tmp.path().join("repo");
16097        std::fs::create_dir_all(&repo).unwrap();
16098        init_repo(&repo);
16099
16100        for (severity, vote, expect) in [
16101            (Severity::Major, ReviewVote::Reject, true),
16102            (Severity::Minor, ReviewVote::Reject, false),
16103            (Severity::Major, ReviewVote::Approve, false),
16104        ] {
16105            let mut round = review_round(false, 1, 1, 1, false, true);
16106            round.reviews = vec![ReviewRecord {
16107                reviewer: 1,
16108                agent: "alpha".to_owned(),
16109                summary: String::new(),
16110                findings: vec![Finding {
16111                    id: "R1-1-1".to_owned(),
16112                    severity,
16113                    file: None,
16114                    line: None,
16115                    title: "t".to_owned(),
16116                    detail: String::new(),
16117                }],
16118                vote: Some(vote),
16119                failed: None,
16120                duration_ms: 0,
16121                attempts: 0,
16122            }];
16123            let mut state = RunState::new(
16124                repo.clone(),
16125                "main".to_owned(),
16126                "deadbeef".to_owned(),
16127                "task".to_owned(),
16128                Config::default(),
16129            );
16130            state.reviews = vec![round];
16131            let mut runner = Runner {
16132                state,
16133                roles: ResolvedRoles {
16134                    implementers: Vec::new(),
16135                    judges: Vec::new(),
16136                    reviewers: Vec::new(),
16137                    fixer: None,
16138                    conductor: conductor(),
16139                    implementer_roster: Vec::new(),
16140                    judge_roster: Vec::new(),
16141                    reviewer_roster: Vec::new(),
16142                },
16143                sem: Arc::new(Semaphore::new(1)),
16144                pause: Pause::new(),
16145                interrupt: Pause::new(),
16146            };
16147            let shell = runner.state.config.shell();
16148            runner
16149                .stop_reviewing("round budget spent", &shell, &repo)
16150                .await
16151                .expect("stop_reviewing");
16152            assert_eq!(runner.state.status, RunStatus::Gating);
16153            assert_eq!(
16154                runner.state.contested_handoff.is_some(),
16155                expect,
16156                "{severity:?} + {vote:?}"
16157            );
16158        }
16159    }
16160
16161    /// The shape the incident this whole fix responds to actually had: the
16162    /// round budget spent, the last round's own e2e blocked on the shared
16163    /// build cache (held here by a live pid — this test process — exactly
16164    /// `cache`'s own unit tests' pattern for "another owner, still alive"
16165    /// without forking a process). `stop_reviewing` must retry it — not
16166    /// silently leave the round looking untouched (the catch-up-only half of
16167    /// the bug), and not read the contention as a red `e2e` and block the
16168    /// run on it (the other half). Called directly, the same way
16169    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
16170    /// above exercises `gate`, so this never needs a real cargo build to
16171    /// reach: the lease is never released, so `with_cache_lease` never gets
16172    /// past acquiring it into anything that would need a real workspace.
16173    #[tokio::test]
16174    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16175        crate::run::pin_test_home();
16176        let home = crate::run::home();
16177
16178        let tmp = tempfile::tempdir().expect("tempdir");
16179        let repo = tmp.path().join("repo");
16180        std::fs::create_dir_all(&repo).unwrap();
16181        init_repo(&repo);
16182        let head = crate::git::rev_parse(&repo, "HEAD")
16183            .await
16184            .expect("rev-parse");
16185        // Unique to this test, so holding its lease cannot collide with
16186        // another test sharing the same process-wide `home`.
16187        let cache_dir = tmp.path().join("target");
16188
16189        let mut config = Config::default();
16190        config.verify.e2e = vec![format!(
16191            "CARGO_TARGET_DIR='{}' test -f README.md",
16192            cache_dir.display()
16193        )];
16194        config.graph.review_rounds = 1;
16195        // Bounded so a regression that does start waiting fails the test in
16196        // seconds, not hangs it.
16197        config.graph.timeout_verify = Some(2);
16198        config.graph.timeout_resource_wait = 2;
16199
16200        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16201        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16202            .expect("no io error acquiring directly")
16203        {
16204            crate::cache::AcquireOutcome::Acquired(g) => g,
16205            crate::cache::AcquireOutcome::Busy(b) => {
16206                panic!("expected the direct acquire to win the lease first: {b:?}")
16207            }
16208        };
16209
16210        let mut state = RunState::new(
16211            repo.clone(),
16212            "main".to_owned(),
16213            head.clone(),
16214            "task".to_owned(),
16215            config,
16216        );
16217        state.candidates = vec![Candidate {
16218            index: 0,
16219            label: 'A',
16220            agent: "alpha".to_owned(),
16221            branch: "does-not-exist".to_owned(),
16222            worktree: repo.clone(),
16223            summary: String::new(),
16224            stat: String::new(),
16225            files: 0,
16226            commits: 0,
16227            empty: false,
16228            failed: None,
16229            verified_noop: None,
16230            duration_ms: 0,
16231            folded: false,
16232        }];
16233        state.tally = Some(Tally {
16234            first_choice: BTreeMap::from([('A', 1)]),
16235            borda: BTreeMap::new(),
16236            winner: 'A',
16237            rankings: 1,
16238            unanimous_initial: true,
16239            deliberated: false,
16240            changed_votes: 0,
16241            unanimous_final: true,
16242            tie_break: None,
16243            judges: 0,
16244            present: 0,
16245            quorum: 0,
16246            met_quorum: true,
16247            uncontested: Some("only candidate A produced a change".to_owned()),
16248        });
16249        // The round budget's last round, deferred: `needs_catchup_run`'s
16250        // other trigger. `stop_reviewing`'s retry machinery must treat this
16251        // exactly like a resource-blocked attempt once it actually runs.
16252        state.reviews = vec![ReviewRound {
16253            round: 1,
16254            head: head.clone(),
16255            verified_head: None,
16256            verified_at: None,
16257            reviews: Vec::new(),
16258            e2e: Vec::new(),
16259            fix: None,
16260            blocking: 1,
16261            answered: 1,
16262            expected: 1,
16263            clean: false,
16264            verify_retried: false,
16265            e2e_deferred: true,
16266            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16267            progressed: false,
16268            vote_split: false,
16269            reconsideration: Vec::new(),
16270            verdict: None,
16271        }];
16272
16273        let mut runner = Runner {
16274            state,
16275            roles: ResolvedRoles {
16276                implementers: Vec::new(),
16277                judges: Vec::new(),
16278                reviewers: Vec::new(),
16279                fixer: None,
16280                conductor: conductor(),
16281                implementer_roster: Vec::new(),
16282                judge_roster: Vec::new(),
16283                reviewer_roster: Vec::new(),
16284            },
16285            sem: Arc::new(Semaphore::new(1)),
16286            pause: Pause::new(),
16287            interrupt: Pause::new(),
16288        };
16289
16290        let shell = runner.state.config.shell();
16291        runner
16292            .stop_reviewing("round budget spent", &shell, &repo)
16293            .await
16294            .expect("stop_reviewing");
16295
16296        let last = runner.state.reviews.last().expect("round record");
16297        assert_eq!(
16298            last.e2e_status(),
16299            E2eStatus::ResourceBlocked,
16300            "the shared cache is still held; the attempt must read as blocked, not deferred or \
16301             failed: {last:?}"
16302        );
16303        assert_eq!(
16304            last.verified_head.as_deref(),
16305            Some(head.as_str()),
16306            "which commit this attempt targeted is known even though nothing finished checking \
16307             it"
16308        );
16309        let first_attempt_at = last
16310            .verified_at
16311            .expect("when this attempt ran is known too");
16312        assert_ne!(
16313            runner.state.status,
16314            RunStatus::Blocked,
16315            "contention is evidence about the machine, not the patch — it must not settle the \
16316             run as blocked: {:?}",
16317            runner.state.status
16318        );
16319        assert!(
16320            !runner
16321                .state
16322                .events
16323                .iter()
16324                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16325            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16326            runner.state.events
16327        );
16328
16329        // The cache is still held: a later reentry must retry the same
16330        // round's verification again — not leave it looking exactly as
16331        // untouched as the first blocked attempt, which is indistinguishable
16332        // from never having tried again at all.
16333        runner
16334            .stop_reviewing("round budget spent", &shell, &repo)
16335            .await
16336            .expect("stop_reviewing retry");
16337        assert_eq!(
16338            runner.state.reviews.len(),
16339            1,
16340            "no new round was started: {:?}",
16341            runner.state.reviews
16342        );
16343        let last = runner.state.reviews.last().expect("round record");
16344        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16345        assert!(
16346            last.verified_at.expect("still known") > first_attempt_at,
16347            "a second reentry must be a fresh attempt, not a stale copy of the first"
16348        );
16349        assert_ne!(runner.state.status, RunStatus::Blocked);
16350
16351        held.release();
16352    }
16353
16354    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
16355    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
16356    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
16357    /// own top-of-function fast path (`review_conclusion`) correctly reads
16358    /// this shape as `None` rather than guessing `Blocked`, and the loop's
16359    /// own `for` range is empty once the round budget is spent, so
16360    /// `review_loop` must retry the check itself rather than silently doing
16361    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
16362    /// above exercises directly, but through `review_loop`'s own entry point
16363    /// this time, proving the wiring between the two rather than just the
16364    /// retry logic in isolation.
16365    #[tokio::test]
16366    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16367        crate::run::pin_test_home();
16368        let home = crate::run::home();
16369
16370        let tmp = tempfile::tempdir().expect("tempdir");
16371        let repo = tmp.path().join("repo");
16372        std::fs::create_dir_all(&repo).unwrap();
16373        init_repo(&repo);
16374        let head = crate::git::rev_parse(&repo, "HEAD")
16375            .await
16376            .expect("rev-parse");
16377        let cache_dir = tmp.path().join("target");
16378
16379        let mut config = Config::default();
16380        config.verify.e2e = vec![format!(
16381            "CARGO_TARGET_DIR='{}' test -f README.md",
16382            cache_dir.display()
16383        )];
16384        config.graph.review_rounds = 1;
16385        config.graph.timeout_verify = Some(2);
16386        config.graph.timeout_resource_wait = 2;
16387
16388        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16389        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16390            .expect("no io error acquiring directly")
16391        {
16392            crate::cache::AcquireOutcome::Acquired(g) => g,
16393            crate::cache::AcquireOutcome::Busy(b) => {
16394                panic!("expected the direct acquire to win the lease first: {b:?}")
16395            }
16396        };
16397
16398        let mut state = RunState::new(
16399            repo.clone(),
16400            "main".to_owned(),
16401            head.clone(),
16402            "task".to_owned(),
16403            config,
16404        );
16405        state.candidates = vec![Candidate {
16406            index: 0,
16407            label: 'A',
16408            agent: "alpha".to_owned(),
16409            branch: "does-not-exist".to_owned(),
16410            worktree: repo.clone(),
16411            summary: String::new(),
16412            stat: String::new(),
16413            files: 0,
16414            commits: 0,
16415            empty: false,
16416            failed: None,
16417            verified_noop: None,
16418            duration_ms: 0,
16419            folded: false,
16420        }];
16421        state.tally = Some(Tally {
16422            first_choice: BTreeMap::from([('A', 1)]),
16423            borda: BTreeMap::new(),
16424            winner: 'A',
16425            rankings: 1,
16426            unanimous_initial: true,
16427            deliberated: false,
16428            changed_votes: 0,
16429            unanimous_final: true,
16430            tie_break: None,
16431            judges: 0,
16432            present: 0,
16433            quorum: 0,
16434            met_quorum: true,
16435            uncontested: Some("only candidate A produced a change".to_owned()),
16436        });
16437        // The exact shape a prior process's `stop_reviewing` would have left
16438        // on disk: the round budget's last round, a real attempt already
16439        // made and already resource-blocked.
16440        state.reviews = vec![ReviewRound {
16441            round: 1,
16442            head: head.clone(),
16443            verified_head: Some(head.clone()),
16444            verified_at: Some(jiff::Timestamp::now()),
16445            reviews: Vec::new(),
16446            e2e: vec![CommandOutcome {
16447                command: format!(
16448                    "CARGO_TARGET_DIR='{}' test -f README.md",
16449                    cache_dir.display()
16450                ),
16451                code: None,
16452                output_tail: "waiting for the shared build cache".to_owned(),
16453                duration_ms: 0,
16454                resource_blocked: true,
16455            }],
16456            fix: None,
16457            blocking: 1,
16458            answered: 1,
16459            expected: 1,
16460            clean: false,
16461            verify_retried: false,
16462            e2e_deferred: false,
16463            e2e_defer_reason: None,
16464            progressed: false,
16465            vote_split: false,
16466            reconsideration: Vec::new(),
16467            verdict: None,
16468        }];
16469
16470        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16471        let mut runner = Runner {
16472            state,
16473            roles: ResolvedRoles {
16474                implementers: Vec::new(),
16475                judges: Vec::new(),
16476                reviewers: Vec::new(),
16477                fixer: None,
16478                conductor: conductor(),
16479                implementer_roster: Vec::new(),
16480                judge_roster: Vec::new(),
16481                reviewer_roster: Vec::new(),
16482            },
16483            sem: Arc::new(Semaphore::new(1)),
16484            pause: Pause::new(),
16485            interrupt: Pause::new(),
16486        };
16487
16488        // The lease is still held throughout, so this reentry's own retry is
16489        // also contended — proving `review_loop` actually tried again (not
16490        // that it happened to succeed) is what the timestamp comparison
16491        // below is for.
16492        runner.review_loop().await.expect("review_loop");
16493
16494        assert_eq!(
16495            runner.state.reviews.len(),
16496            1,
16497            "no new round was started on top of the unresolved one: {:?}",
16498            runner.state.reviews
16499        );
16500        let last = &runner.state.reviews[0];
16501        assert_eq!(
16502            last.e2e_status(),
16503            E2eStatus::ResourceBlocked,
16504            "still contended: {last:?}"
16505        );
16506        assert!(
16507            last.verified_at.expect("still known") > first_attempt_at,
16508            "review_loop must have actually retried the check, not left it exactly as found"
16509        );
16510        assert_ne!(
16511            runner.state.status,
16512            RunStatus::Blocked,
16513            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16514            runner.state.status
16515        );
16516
16517        held.release();
16518    }
16519
16520    #[tokio::test]
16521    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
16522        crate::run::pin_test_home();
16523        for status in [RunStatus::Blocked, RunStatus::Gating] {
16524            let mut runner = runner_at(status);
16525            runner.state.config.graph.land = true;
16526            runner.state.gate_ran = true;
16527            runner.state.merge = Some(MergeOutcome {
16528                mode: MergeMode::Pr,
16529                ok: false,
16530                detail: "the fixer produced no commit".into(),
16531                empty: false,
16532            });
16533            let url = "https://example.invalid/o/r/pull/1";
16534            runner.state.pr = Some(crate::run::PrRecord {
16535                url: url.into(),
16536                number: 1,
16537                state: "open".into(),
16538                checks: "red".into(),
16539                round: 1,
16540                rounds: 1,
16541                red_at_merge: Vec::new(),
16542            });
16543            runner.state.save().unwrap();
16544            // The nonexistent repository makes observation fail before a forge
16545            // command can run. We still must reach land, rather than silently
16546            // return from the graph with its old gate and merge records.
16547            runner.execute_graph().await.unwrap();
16548            assert_eq!(runner.state.status, RunStatus::Blocked);
16549            assert!(runner.state.land_sync_pending);
16550            assert!(runner.state.events.iter().any(|event|
16551                event.node == "land" && event.message == format!("watching {url}")));
16552            assert_eq!(
16553                runner.state.merge.as_ref().unwrap().detail,
16554                "the fixer produced no commit"
16555            );
16556        }
16557    }
16558
16559    #[tokio::test]
16560    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16561        crate::run::pin_test_home();
16562        let tmp = tempfile::tempdir().expect("tempdir");
16563        let repo = tmp.path().join("repo");
16564        std::fs::create_dir_all(&repo).unwrap();
16565        init_repo(&repo);
16566
16567        let mut config = Config::default();
16568        config.merge.mode = MergeMode::Pr;
16569        config.graph.land = true;
16570        config.graph.land_approval = false;
16571
16572        let mut state = RunState::new(
16573            repo.clone(),
16574            "main".to_owned(),
16575            "deadbeef".to_owned(),
16576            "task".to_owned(),
16577            config,
16578        );
16579        state.candidates = vec![Candidate {
16580            index: 0,
16581            label: 'A',
16582            agent: "alpha".to_owned(),
16583            branch: "does-not-exist".to_owned(),
16584            worktree: repo.clone(),
16585            summary: String::new(),
16586            stat: String::new(),
16587            files: 0,
16588            commits: 0,
16589            empty: false,
16590            failed: None,
16591            verified_noop: None,
16592            duration_ms: 0,
16593            folded: false,
16594        }];
16595        state.tally = Some(Tally {
16596            first_choice: BTreeMap::from([('A', 1)]),
16597            borda: BTreeMap::new(),
16598            winner: 'A',
16599            rankings: 1,
16600            unanimous_initial: true,
16601            deliberated: false,
16602            changed_votes: 0,
16603            unanimous_final: true,
16604            tie_break: None,
16605            judges: 0,
16606            present: 0,
16607            quorum: 0,
16608            met_quorum: true,
16609            uncontested: Some("only candidate A produced a change".to_owned()),
16610        });
16611        state.reviews = vec![ReviewRound {
16612            round: 1,
16613            head: "deadbeef".to_owned(),
16614            verified_head: None,
16615            verified_at: None,
16616            reviews: Vec::new(),
16617            e2e: Vec::new(),
16618            fix: None,
16619            blocking: 0,
16620            answered: 0,
16621            expected: 0,
16622            clean: true,
16623            verify_retried: false,
16624            e2e_deferred: false,
16625            e2e_defer_reason: None,
16626            progressed: false,
16627            vote_split: false,
16628            reconsideration: Vec::new(),
16629            verdict: None,
16630        }];
16631        state.gate = vec![CommandOutcome {
16632            command: "test".to_owned(),
16633            code: Some(0),
16634            output_tail: String::new(),
16635            duration_ms: 0,
16636            resource_blocked: false,
16637        }];
16638        state.gate_ran = true;
16639        // A first pass through `merge` already pushed and opened this pull
16640        // request; `status` is `Landing` because a previous call into `land`
16641        // parked or was interrupted before it reached a terminal outcome.
16642        state.status = RunStatus::Landing;
16643        state.merge = Some(MergeOutcome {
16644            mode: MergeMode::Pr,
16645            ok: true,
16646            detail: "https://example.invalid/x/y/pull/1".to_owned(),
16647            empty: false,
16648        });
16649
16650        // The Landing-resume shortcut calls `run_land` directly rather than
16651        // through `merge`, which is exactly the call site that used to skip
16652        // `settle_questions` - see the fixture below.
16653        ask_test_home();
16654        // A real resume loads a persisted run; driver ownership is checked
16655        // against that record before the Landing shortcut is entered.
16656        state.save().expect("save landing run");
16657        let store = ask::Questions::open();
16658        let q = ask_open_question(&store, &state.id);
16659
16660        let mut runner = Runner {
16661            state,
16662            roles: ResolvedRoles {
16663                implementers: Vec::new(),
16664                judges: Vec::new(),
16665                reviewers: Vec::new(),
16666                fixer: None,
16667                conductor: conductor(),
16668                implementer_roster: Vec::new(),
16669                judge_roster: Vec::new(),
16670                reviewer_roster: Vec::new(),
16671            },
16672            sem: Arc::new(Semaphore::new(1)),
16673            pause: Pause::new(),
16674            interrupt: Pause::new(),
16675        };
16676
16677        // `execute`, not `merge` directly: the Landing-resume shortcut lives
16678        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
16679        // exactly because `review_loop` would otherwise clobber the marker
16680        // first.
16681        runner.execute().await.expect("execute");
16682
16683        assert_eq!(
16684            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16685            Some("https://example.invalid/x/y/pull/1"),
16686            "reentry must not push again or open a second pull request over the \
16687             one `land` is already watching"
16688        );
16689        assert_ne!(
16690            runner.state.status,
16691            RunStatus::Landing,
16692            "land could not actually reach the fake pull request, so it must \
16693             have given up rather than left the run silently parked forever"
16694        );
16695        // `land` could not reach the fake pull request, so it gave up into
16696        // `Blocked` - still resumable, so the question must not have been
16697        // swept just because this branch now also calls `settle_questions`.
16698        assert_eq!(runner.state.status, RunStatus::Blocked);
16699        assert!(
16700            store.get(&q.id).unwrap().status.open(),
16701            "Blocked is still alive; settle_questions must have been a no-op here"
16702        );
16703    }
16704
16705    fn state_with_round(round: ReviewRound) -> RunState {
16706        let mut s = RunState::new(
16707            PathBuf::from("/repo"),
16708            "main".to_owned(),
16709            "abc1234".to_owned(),
16710            "add retries".to_owned(),
16711            Config::default(),
16712        );
16713        s.reviews = vec![round];
16714        s
16715    }
16716
16717    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16718        crate::verdict::Finding {
16719            id: id.to_owned(),
16720            severity,
16721            file: None,
16722            line: None,
16723            title: title.to_owned(),
16724            detail: String::new(),
16725        }
16726    }
16727
16728    #[test]
16729    fn pr_body_names_open_findings_and_declined_ones() {
16730        let round = ReviewRound {
16731            round: 2,
16732            head: "deadbee".to_owned(),
16733            verified_head: None,
16734            verified_at: None,
16735            reviews: vec![ReviewRecord {
16736                attempts: 0,
16737                reviewer: 1,
16738                agent: "alpha".to_owned(),
16739                summary: String::new(),
16740                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16741                vote: None,
16742                failed: None,
16743                duration_ms: 0,
16744            }],
16745            e2e: vec![CommandOutcome {
16746                command: "cargo test".to_owned(),
16747                code: Some(0),
16748                output_tail: String::new(),
16749                duration_ms: 0,
16750                resource_blocked: false,
16751            }],
16752            verify_retried: false,
16753            e2e_deferred: false,
16754            e2e_defer_reason: None,
16755            fix: Some(FixRecord {
16756                agent: "alpha".to_owned(),
16757                addressed: Vec::new(),
16758                rejected: vec![crate::verdict::Rejection {
16759                    id: "R1-1-1".to_owned(),
16760                    why: "not reachable from any caller".to_owned(),
16761                }],
16762                notes: String::new(),
16763                committed: true,
16764                failed: None,
16765                duration_ms: 0,
16766                continuation: None,
16767            }),
16768            blocking: 0,
16769            answered: 1,
16770            expected: 1,
16771            clean: false,
16772            progressed: true,
16773            vote_split: false,
16774            reconsideration: Vec::new(),
16775            verdict: None,
16776        };
16777        let state = state_with_round(round);
16778        let body = pr_message(&state, 'A').body;
16779
16780        assert!(body.contains("add retries"), "the task must still be there");
16781        assert!(body.contains("R2-1-1"), "{body}");
16782        assert!(!body.contains("unused import"), "{body}");
16783        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16784        assert!(
16785            !body.contains("not reachable from any caller"),
16786            "the reason it was declined: {body}"
16787        );
16788    }
16789
16790    #[test]
16791    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16792        let round = ReviewRound {
16793            round: 1,
16794            head: "deadbee".to_owned(),
16795            verified_head: None,
16796            verified_at: None,
16797            reviews: vec![ReviewRecord {
16798                attempts: 0,
16799                reviewer: 1,
16800                agent: "alpha".to_owned(),
16801                summary: String::new(),
16802                findings: Vec::new(),
16803                vote: None,
16804                failed: None,
16805                duration_ms: 0,
16806            }],
16807            e2e: Vec::new(),
16808            verify_retried: false,
16809            e2e_deferred: false,
16810            e2e_defer_reason: None,
16811            fix: None,
16812            blocking: 0,
16813            answered: 1,
16814            expected: 1,
16815            clean: true,
16816            progressed: false,
16817            vote_split: false,
16818            reconsideration: Vec::new(),
16819            verdict: None,
16820        };
16821        let state = state_with_round(round);
16822        let body = pr_message(&state, 'A').body;
16823        assert!(!body.contains("Open review findings"), "{body}");
16824        assert!(!body.contains("Declined"), "{body}");
16825    }
16826
16827    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16828        let mut state = RunState::new(
16829            PathBuf::from("/repo"),
16830            "main".to_owned(),
16831            "abc1234".to_owned(),
16832            instruction.to_owned(),
16833            Config::default(),
16834        );
16835        state.candidates.push(Candidate {
16836            index: 0,
16837            label: 'A',
16838            agent: "alpha".to_owned(),
16839            branch: "magi/x/A".to_owned(),
16840            worktree: PathBuf::from("/wt"),
16841            summary: summary.to_owned(),
16842            stat: String::new(),
16843            files: 1,
16844            commits: 1,
16845            empty: false,
16846            failed: None,
16847            verified_noop: None,
16848            folded: false,
16849            duration_ms: 0,
16850        });
16851        state
16852    }
16853
16854    fn review_state(subjects: &[&str]) -> RunState {
16855        let mut state = state_with_summary(
16856            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16857            "",
16858        );
16859        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16860        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16861        state
16862    }
16863
16864    /// A branch rebased onto a main that moved past the recorded
16865    /// `base_commit` is titled from its own first commit, never main's.
16866    #[tokio::test]
16867    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16868        ask_test_home();
16869        let tmp = tempfile::tempdir().unwrap();
16870        let repo = tmp.path().join("repo");
16871        std::fs::create_dir_all(&repo).unwrap();
16872        init_repo(&repo);
16873        let origin = tmp.path().join("origin.git");
16874        let g = |dir: &Path, args: &[&str]| {
16875            let out = std::process::Command::new("git")
16876                .args(args)
16877                .current_dir(dir)
16878                .quiet()
16879                .output()
16880                .expect("spawn git");
16881            assert!(
16882                out.status.success(),
16883                "git {args:?}: {}",
16884                String::from_utf8_lossy(&out.stderr)
16885            );
16886        };
16887        g(
16888            tmp.path(),
16889            &[
16890                "clone",
16891                "--bare",
16892                "-q",
16893                repo.to_str().unwrap(),
16894                origin.to_str().unwrap(),
16895            ],
16896        );
16897        g(
16898            &repo,
16899            &["remote", "add", "origin", origin.to_str().unwrap()],
16900        );
16901        let c1 = git::rev_parse(&repo, "main").await.unwrap();
16902
16903        // Main moves on; the branch is built on top of the new main.
16904        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16905        g(&repo, &["add", "-A"]);
16906        g(
16907            &repo,
16908            &["commit", "-q", "-m", "chore(deps): update a crate"],
16909        );
16910        g(&repo, &["push", "-q", "origin", "main"]);
16911        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16912        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16913        g(&repo, &["add", "-A"]);
16914        g(
16915            &repo,
16916            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16917        );
16918        g(&repo, &["checkout", "-q", "main"]);
16919
16920        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16921        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16922        // Without a readable tracking ref the recorded base's merge base is used.
16923        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16924        assert_eq!(fallback, c1);
16925
16926        let mut state = review_state(&[
16927            "chore(deps): update a crate",
16928            "fix(daemon): apply a chosen action",
16929        ]);
16930        state.repo = repo.clone();
16931        state.base_branch = "main".to_owned();
16932        state.base_commit = c1;
16933        refresh_reviewed_commits(&mut state, "feat/own").await;
16934        assert_eq!(
16935            state.reviewed_commits,
16936            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16937        );
16938        assert_eq!(
16939            review_title(&state).as_deref(),
16940            Some("fix(daemon): apply a chosen action")
16941        );
16942        assert_eq!(
16943            leaked_subjects(&state, "feat/own").await,
16944            Some(vec!["chore(deps): update a crate".to_owned()])
16945        );
16946        // A stale tracking ref is still used when the fetch fails, but the
16947        // leak list is withheld.
16948        state.config.merge.remote = "nowhere".to_owned();
16949        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16950    }
16951
16952    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
16953    /// with `origin/main` one commit behind it, and a summarizing agent that
16954    /// prints whatever `reply` holds.
16955    struct SummaryFixture {
16956        _tmp: tempfile::TempDir,
16957        repo: PathBuf,
16958        reply: PathBuf,
16959        runner: Runner,
16960    }
16961
16962    fn summary_fixture() -> SummaryFixture {
16963        ask_test_home();
16964        let tmp = tempfile::tempdir().unwrap();
16965        let repo = tmp.path().join("repo");
16966        std::fs::create_dir_all(&repo).unwrap();
16967        init_repo(&repo);
16968        let origin = tmp.path().join("origin.git");
16969        let g = |args: &[&str]| {
16970            let out = std::process::Command::new("git")
16971                .args(args)
16972                .current_dir(&repo)
16973                .quiet()
16974                .output()
16975                .expect("spawn git");
16976            assert!(
16977                out.status.success(),
16978                "git {args:?}: {}",
16979                String::from_utf8_lossy(&out.stderr)
16980            );
16981        };
16982        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16983        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16984        g(&["push", "-q", "origin", "main"]);
16985        g(&["checkout", "-q", "-b", "feat/own"]);
16986        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16987        g(&["add", "-A"]);
16988        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16989        let reply = tmp.path().join("reply.txt");
16990        std::fs::write(
16991            &reply,
16992            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16993        )
16994        .unwrap();
16995        let mut runner = runner_at(RunStatus::Gating);
16996        let mut state = review_state(&["feat(own): add the own file"]);
16997        state.repo = repo.clone();
16998        state.base_branch = "main".to_owned();
16999        state.candidates[0].branch = "feat/own".to_owned();
17000        state.candidates[0].worktree = repo.clone();
17001        let mut author = spec("alpha");
17002        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17003        author
17004            .env
17005            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17006        state.config.agents = vec![author];
17007        runner.state = state;
17008        SummaryFixture {
17009            _tmp: tmp,
17010            repo,
17011            reply,
17012            runner,
17013        }
17014    }
17015
17016    async fn posted(f: &mut SummaryFixture) -> PrMessage {
17017        let winner = f.runner.state.candidates[0].clone();
17018        f.runner
17019            .guarded_pr_message(&winner, None, true)
17020            .await
17021            .unwrap()
17022            .expect("an English description never parks")
17023    }
17024
17025    fn clean_round_with_minor_findings() -> ReviewRound {
17026        let mut round = passing_round(2, None);
17027        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17028        round.reviews = vec![ReviewRecord {
17029            attempts: 0,
17030            reviewer: 1,
17031            agent: "alpha".into(),
17032            summary: "内部レビュー".into(),
17033            findings: vec![
17034                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17035                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17036            ],
17037            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17038            failed: None,
17039            duration_ms: 0,
17040        }];
17041        round
17042    }
17043
17044    #[tokio::test]
17045    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17046        for review_only in [false, true] {
17047            let mut f = summary_fixture();
17048            if !review_only {
17049                f.runner.state.candidates[0].agent = "alpha".into();
17050            }
17051            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17052            assert!(f.runner.state.reviews[0].clean);
17053            assert!(f.runner.state.reviews[0].fix.is_none());
17054            assert!(f.runner.state.open_findings().is_empty());
17055            assert_eq!(
17056                crate::pr_outcome::required_finding_ids(&f.runner.state),
17057                ["R2-1-1", "R2-1-2"]
17058            );
17059            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17060            let evidence = crate::pr_outcome::summary_evidence(
17061                &f.runner.state,
17062                crate::pr_outcome::FinalHead::At(&head),
17063            );
17064            assert!(
17065                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17066            );
17067            // The ordinary reply omits both IDs: no cached description is adopted.
17068            let omitted = posted(&mut f).await;
17069            assert!(omitted.body.contains("No description"));
17070            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17071            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17072            assert!(f.runner.state.change_summary.is_none());
17073            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17074            let described = posted(&mut f).await;
17075            assert!(described.body.contains("improve documentation for callers"));
17076            assert!(described.body.contains("remaining edge case"));
17077            assert!(described.body.contains("## Open review findings"));
17078            assert!(!described.body.contains("未修正"));
17079        }
17080    }
17081
17082    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17083        let mut f = summary_fixture();
17084        if !review_only {
17085            f.runner.state.candidates[0].agent = "alpha".into();
17086        }
17087        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17088        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17089        fixed
17090            .fix
17091            .as_mut()
17092            .unwrap()
17093            .rejected
17094            .push(crate::verdict::Rejection {
17095                id: "R1-1-1".into(),
17096                why: "呼び出されないコードのため辞退".into(),
17097            });
17098        let mut clean = clean_round_with_minor_findings();
17099        clean.verified_head = Some(head.clone());
17100        f.runner.state.reviews = vec![fixed, clean];
17101        ok_gate(&mut f.runner.state, Some(&head));
17102        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17103        f.runner.state.config.agents.push(scripted("beta", reply));
17104        f.runner.state.config.roles.language_judge =
17105            Some(crate::config::AgentChoice::One("beta".into()));
17106        f
17107    }
17108
17109    #[tokio::test]
17110    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17111        for review_only in [false, true] {
17112            let mut f = rewrite_fixture_with_findings(review_only,
17113                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17114            let m = posted(&mut f).await;
17115            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17116            let expected =
17117                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17118            assert!(m.body.contains(&expected));
17119            assert!(m.body.contains("## Review rounds"));
17120            assert!(
17121                m.body
17122                    .contains("Gate (`verify.gate`): passed on the final commit")
17123            );
17124            assert!(
17125                m.body
17126                    .contains("Tests (`verify.e2e`): passed on the final commit")
17127            );
17128            let c = f.runner.state.change_summary.as_ref().unwrap();
17129            assert!(
17130                m.body
17131                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17132            );
17133            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17134            assert!(crate::pr_outcome::covers_required_findings(
17135                &f.runner.state,
17136                &m.body
17137            ));
17138            assert!(m.body.contains("documentation remains incomplete"));
17139            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17140            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17141            assert!(f.runner.state.github_text.is_none());
17142        }
17143    }
17144
17145    #[tokio::test]
17146    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17147        for review_only in [false, true] {
17148            for reply in [
17149                "TITLE: fix: public description\nA short English description.",
17150                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17151            ] {
17152                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17153                let winner = f.runner.state.candidates[0].clone();
17154                let posted = f
17155                    .runner
17156                    .guarded_pr_message(&winner, None, true)
17157                    .await
17158                    .unwrap();
17159                assert!(posted.is_none());
17160                assert!(f.runner.state.parked);
17161                let withheld = f.runner.state.github_text.as_ref().unwrap();
17162                assert!(
17163                    withheld.rewrite.is_none(),
17164                    "an incomplete rewrite must not be selectable"
17165                );
17166                let q = ask::Questions::open()
17167                    .get(withheld.question.as_ref().unwrap())
17168                    .unwrap();
17169                assert!(
17170                    !q.choices
17171                        .contains(&crate::github_text::USE_REWRITE.to_owned())
17172                );
17173                assert!(
17174                    q.detail.contains("R1-1-1")
17175                        && q.detail.contains("R2-1-1")
17176                        && q.detail.contains("R2-1-2")
17177                );
17178            }
17179        }
17180    }
17181
17182    #[tokio::test]
17183    async fn japanese_reports_stay_local_in_both_public_run_modes() {
17184        for review_only in [false, true] {
17185            let mut f = summary_fixture();
17186            if !review_only {
17187                f.runner.state.candidates[0].agent = "alpha".into();
17188            }
17189            f.runner.state.instruction = "内部タスクの説明".into();
17190            f.runner.state.candidates[0].summary = "内部の候補報告".into();
17191            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17192            f.runner.state.reviews = vec![
17193                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17194                round_with_notes(2, Some("二回目の内部修正報告です。")),
17195                passing_round(3, Some(&head)),
17196            ];
17197            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17198            ok_gate(&mut f.runner.state, Some(&head));
17199            let captured = f.repo.join("public-prompt.txt");
17200            let author = &mut f.runner.state.config.agents[0];
17201            author.command = vec![
17202                "sh".into(),
17203                "-c".into(),
17204                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17205            ];
17206            author.env.insert(
17207                "PROMPT_FILE".into(),
17208                captured.to_string_lossy().into_owned(),
17209            );
17210            let m = posted(&mut f).await;
17211            assert!(m.body.contains("Adds `own.txt`"));
17212            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17213            assert!(m.body.contains("R1-1-1"));
17214            assert!(
17215                m.body
17216                    .contains("Gate (`verify.gate`): passed on the final commit")
17217            );
17218            assert!(
17219                m.body
17220                    .contains("Tests (`verify.e2e`): passed on the final commit")
17221            );
17222            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17223            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17224            let prompt = std::fs::read_to_string(captured).unwrap();
17225            assert!(prompt.contains(&format!(
17226                "git diff {} {head}",
17227                f.runner.state.change_summary.as_ref().unwrap().base
17228            )));
17229            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17230            assert_eq!(
17231                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17232                "二回目の内部修正報告です。"
17233            );
17234        }
17235    }
17236
17237    #[tokio::test]
17238    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17239        for review_only in [false, true] {
17240            let mut f = summary_fixture();
17241            if !review_only {
17242                f.runner.state.candidates[0].agent = "alpha".into();
17243            }
17244            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17245            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17246            std::fs::write(&f.reply, "no usable description").unwrap();
17247            let failed = posted(&mut f).await;
17248            assert!(
17249                failed
17250                    .body
17251                    .contains("No description of what this change does could be generated")
17252            );
17253            assert!(failed.body.is_ascii());
17254            assert!(f.runner.state.github_text.is_none());
17255            assert!(f.runner.state.change_summary.is_none());
17256            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17257            let evidence = crate::pr_outcome::summary_evidence(
17258                &f.runner.state,
17259                crate::pr_outcome::FinalHead::At(&head),
17260            );
17261            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17262            assert!(
17263                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17264                    .exists()
17265            );
17266            f.runner.state =
17267                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17268            std::fs::write(
17269                &f.reply,
17270                "TITLE: feat(own): add the own file\nAdds the committed own file.",
17271            )
17272            .unwrap();
17273            let retried = posted(&mut f).await;
17274            assert!(retried.body.contains("Adds the committed own file."));
17275            assert_eq!(
17276                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17277                "日本語の修正報告"
17278            );
17279        }
17280    }
17281
17282    #[tokio::test]
17283    async fn same_head_verification_and_findings_invalidate_public_prose() {
17284        let mut f = summary_fixture();
17285        posted(&mut f).await;
17286        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17287        let original = f
17288            .runner
17289            .state
17290            .change_summary
17291            .as_ref()
17292            .unwrap()
17293            .evidence
17294            .clone();
17295        ok_gate(&mut f.runner.state, Some(&head));
17296        std::fs::write(
17297            &f.reply,
17298            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17299        )
17300        .unwrap();
17301        let verified = posted(&mut f).await;
17302        assert!(verified.body.contains("now verified"));
17303        assert_ne!(
17304            f.runner.state.change_summary.as_ref().unwrap().evidence,
17305            original
17306        );
17307        let mut round = round_with_notes(1, Some("内部報告"));
17308        round.reviews.push(ReviewRecord {
17309            attempts: 0,
17310            reviewer: 1,
17311            agent: "alpha".into(),
17312            summary: "内部レビュー".into(),
17313            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17314            vote: None,
17315            failed: None,
17316            duration_ms: 0,
17317        });
17318        round
17319            .fix
17320            .as_mut()
17321            .unwrap()
17322            .rejected
17323            .push(crate::verdict::Rejection {
17324                id: "R1-1-1".into(),
17325                why: "呼び出し元から到達できない".into(),
17326            });
17327        f.runner.state.reviews = vec![round];
17328        // A description omitting the required ID is unusable, even with a title.
17329        let omitted = posted(&mut f).await;
17330        assert!(omitted.body.contains("No description"));
17331        assert!(omitted.body.contains("R1-1-1"));
17332        assert!(!omitted.body.contains("到達"));
17333        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17334        let described = posted(&mut f).await;
17335        assert!(described.body.contains("unreachable from callers"));
17336        assert!(described.body.contains("R1-1-1"));
17337        assert!(!described.body.contains("到達"));
17338        let evidence = f
17339            .runner
17340            .state
17341            .change_summary
17342            .as_ref()
17343            .unwrap()
17344            .evidence
17345            .clone();
17346        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17347        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17348        assert!(posted(&mut f).await.body.contains("rationale updated"));
17349        assert_ne!(
17350            f.runner.state.change_summary.as_ref().unwrap().evidence,
17351            evidence
17352        );
17353        // Old serialized summaries have no evidence fingerprint and must regenerate.
17354        f.runner
17355            .state
17356            .change_summary
17357            .as_mut()
17358            .unwrap()
17359            .evidence
17360            .clear();
17361        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17362        assert!(posted(&mut f).await.body.contains("current specification"));
17363    }
17364
17365    #[tokio::test]
17366    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17367        let mut f = summary_fixture();
17368        let m = posted(&mut f).await;
17369        assert_eq!(m.title, "feat(own): add the own file");
17370        assert!(
17371            m.body
17372                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17373            "{}",
17374            m.body
17375        );
17376        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17377        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17378        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17379        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17380        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17381        let kept = f.runner.state.change_summary.as_ref().unwrap();
17382        assert_eq!(
17383            (kept.head.as_str(), kept.base.as_str()),
17384            (head.as_str(), base.as_str())
17385        );
17386        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17387    }
17388
17389    #[tokio::test]
17390    async fn review_only_summary_receives_the_configured_github_language() {
17391        let mut f = summary_fixture();
17392        f.runner.state.config.graph.github_language = "ja".into();
17393        let captured = f.repo.join("summary-prompt.txt");
17394        let author = &mut f.runner.state.config.agents[0];
17395        author.command = vec![
17396            "sh".into(),
17397            "-c".into(),
17398            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17399        ];
17400        author.env.insert(
17401            "PROMPT_FILE".into(),
17402            captured.to_string_lossy().into_owned(),
17403        );
17404        std::fs::write(
17405            &f.reply,
17406            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17407        )
17408        .unwrap();
17409        let m = posted(&mut f).await;
17410        let prompt = std::fs::read_to_string(captured).unwrap();
17411        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17412        assert!(!prompt.contains("in English"));
17413        assert_eq!(m.title, "feat(own): ファイルを追加");
17414        assert!(m.body.contains("own.txt を追加します。"));
17415    }
17416
17417    #[tokio::test]
17418    async fn review_fixes_come_after_the_whole_change_summary() {
17419        let mut f = summary_fixture();
17420        f.runner.state.reviews = vec![
17421            round_with_notes(1, Some("handled the PATH case")),
17422            round_with_notes(2, None),
17423        ];
17424        let m = posted(&mut f).await;
17425        let summary = m.body.find("## Summary").expect("summary");
17426        let fixes = m.body.find("## Review rounds").expect("review rounds");
17427        assert!(summary < fixes, "{}", m.body);
17428        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17429        assert!(m.body.contains("## Final verification"), "{}", m.body);
17430    }
17431
17432    #[tokio::test]
17433    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17434        let mut f = summary_fixture();
17435        let first = posted(&mut f).await;
17436        let first_head = f.runner.state.change_summary.clone().unwrap().head;
17437        // The same head reuses the record: a different reply is not read.
17438        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17439        let again = posted(&mut f).await;
17440        assert_eq!((again.title, again.body), (first.title, first.body));
17441
17442        let g = |args: &[&str]| {
17443            let out = std::process::Command::new("git")
17444                .args(args)
17445                .current_dir(&f.repo)
17446                .quiet()
17447                .output()
17448                .expect("spawn git");
17449            assert!(out.status.success(), "git {args:?}");
17450        };
17451        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17452        g(&["add", "-A"]);
17453        g(&["commit", "-q", "-m", "fix(own): handle more"]);
17454        let second = posted(&mut f).await;
17455        let kept = f.runner.state.change_summary.clone().unwrap();
17456        assert_ne!(kept.head, first_head);
17457        assert!(second.body.contains("Not read yet."), "{}", second.body);
17458        assert!(!second.body.contains("so the first summary is read"));
17459        assert_eq!(second.title, "feat(own): other");
17460    }
17461
17462    #[tokio::test]
17463    async fn an_unavailable_summary_is_stated_and_never_invented() {
17464        let mut f = summary_fixture();
17465        std::fs::write(&f.reply, "no title line here").unwrap();
17466        let m = posted(&mut f).await;
17467        assert!(
17468            m.body
17469                .contains("No description of what this change does could be generated"),
17470            "{}",
17471            m.body
17472        );
17473        assert!(m.body.contains("internal reports remain in local records"));
17474        assert!(f.runner.state.change_summary.is_none());
17475        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17476        // A preview asks nobody and claims nothing.
17477        let winner = f.runner.state.candidates[0].clone();
17478        let preview = f
17479            .runner
17480            .guarded_pr_message(&winner, None, false)
17481            .await
17482            .unwrap()
17483            .unwrap();
17484        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17485    }
17486
17487    #[test]
17488    fn pr_message_review_single_commit_uses_its_subject() {
17489        let state = review_state(&["feat(nats): per-role user"]);
17490        let m = pr_message(&state, 'A');
17491        assert_eq!(m.title, "feat(nats): per-role user");
17492        assert!(!m.body.contains("Review the work already"), "{}", m.body);
17493        assert!(m.body.contains("## Commits under review"), "{}", m.body);
17494    }
17495
17496    #[test]
17497    fn pr_message_review_multi_commit_takes_the_oldest() {
17498        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17499        let m = pr_message(&state, 'A');
17500        assert_eq!(m.title, "feat: the change");
17501        for s in ["feat: the change", "fix: typo", "fix: again"] {
17502            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17503        }
17504    }
17505
17506    #[test]
17507    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17508        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17509            let mut state = review_state(&[first, "fix: later fixup"]);
17510            state.candidates[0].branch = "機能/ログイン".to_owned();
17511            let m = pr_message(&state, 'A');
17512            assert!(
17513                m.title.starts_with("chore: land candidate A of run"),
17514                "{}",
17515                m.title
17516            );
17517        }
17518    }
17519
17520    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17521        BranchFacts {
17522            commits: commits
17523                .iter()
17524                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17525                .collect(),
17526            stat: stat.to_owned(),
17527        }
17528    }
17529
17530    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17531        let mut r = review_round(true, 0, 1, 1, true, true);
17532        r.round = round;
17533        r.fix = notes.map(|n| FixRecord {
17534            agent: "fixer".to_owned(),
17535            addressed: Vec::new(),
17536            rejected: Vec::new(),
17537            notes: n.to_owned(),
17538            committed: true,
17539            failed: None,
17540            duration_ms: 0,
17541            continuation: None,
17542        });
17543        r
17544    }
17545
17546    #[test]
17547    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17548        let state = review_state(&["ignored"]);
17549        let f = facts(
17550            &[
17551                (
17552                    "fix(login): resolve PATH on macOS",
17553                    "Login shells skip rc files.",
17554                ),
17555                ("fix: address review", ""),
17556            ],
17557            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17558        );
17559        let m = pr_message_with(&state, 'A', Some(&f));
17560        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17561        assert!(
17562            m.body
17563                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
17564            "{}",
17565            m.body
17566        );
17567        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17568        assert!(m.body.contains("## Diff stat"), "{}", m.body);
17569        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17570        for banned in [
17571            "Review the work already",
17572            "no task statement",
17573            "Original task",
17574        ] {
17575            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17576        }
17577        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17578    }
17579
17580    #[test]
17581    fn pr_message_review_truncates_a_huge_first_commit_body() {
17582        let state = review_state(&["ignored"]);
17583        let f = facts(
17584            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17585            "s",
17586        );
17587        let m = pr_message_with(&state, 'A', Some(&f));
17588        assert!(m.body.len() < 30_000, "{}", m.body.len());
17589        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17590        assert!(
17591            m.body.contains("- ... 1 more commit(s)"),
17592            "{}",
17593            m.body.len()
17594        );
17595        assert!(m.body.ends_with("magi:candidate-a\n"));
17596    }
17597
17598    #[test]
17599    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17600        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17601        assert_eq!(m.title, "feat: x");
17602        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17603        assert!(!m.body.contains("Diff stat"), "{}", m.body);
17604        assert!(!m.body.contains("no task statement"), "{}", m.body);
17605    }
17606
17607    #[test]
17608    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17609        let mut state = review_state(&["日本語の件名"]);
17610        state.candidates[0].branch = "fix/macos-login-path".to_owned();
17611        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17612        state.candidates[0].branch = "機能/ログイン".to_owned();
17613        assert!(
17614            pr_message(&state, 'A')
17615                .title
17616                .starts_with("chore: land candidate A")
17617        );
17618    }
17619
17620    #[test]
17621    fn pr_message_review_fixes_survive_a_clean_final_round() {
17622        let mut state = review_state(&["feat: x"]);
17623        state.reviews = vec![
17624            round_with_notes(1, Some("handled the PATH case")),
17625            round_with_notes(2, None),
17626        ];
17627        let body = pr_message(&state, 'A').body;
17628        assert!(
17629            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17630            "{body}"
17631        );
17632        assert!(body.contains("### Round 1"), "{body}");
17633
17634        state.reviews = vec![
17635            round_with_notes(1, Some("first fix")),
17636            round_with_notes(2, Some("")),
17637            round_with_notes(3, Some("second fix")),
17638            round_with_notes(4, None),
17639        ];
17640        let body = pr_message(&state, 'A').body;
17641        assert!(
17642            body.contains("### Round 1") && !body.contains("first fix"),
17643            "{body}"
17644        );
17645        assert!(
17646            body.contains("### Round 3") && !body.contains("second fix"),
17647            "{body}"
17648        );
17649        assert!(body.contains("### Round 2"), "{body}");
17650    }
17651
17652    const FINAL: &str = "abcdef1234567890";
17653
17654    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17655        let mut r = round_with_notes(round, None);
17656        r.verified_head = verified.map(str::to_owned);
17657        r
17658    }
17659
17660    fn ok_gate(state: &mut RunState, head: Option<&str>) {
17661        state.gate = vec![CommandOutcome {
17662            command: "gate".to_owned(),
17663            code: Some(0),
17664            output_tail: String::new(),
17665            duration_ms: 0,
17666            resource_blocked: false,
17667        }];
17668        state.gate_ran = true;
17669        state.gate_head = head.map(str::to_owned);
17670    }
17671
17672    fn body_at(state: &RunState) -> String {
17673        pr_message_raw(
17674            state,
17675            'A',
17676            None,
17677            crate::pr_outcome::FinalHead::At(FINAL),
17678            ChangeNote::NotAsked,
17679        )
17680        .body
17681    }
17682
17683    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17684
17685    #[test]
17686    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17687        let mut state = review_state(&["feat: x"]);
17688        state.reviews = vec![
17689            round_with_notes(1, Some(SANDBOXED)),
17690            passing_round(2, Some(FINAL)),
17691        ];
17692        ok_gate(&mut state, Some(FINAL));
17693        let body = body_at(&state);
17694        assert!(!body.contains("Handled the PATH case."), "{body}");
17695        assert!(!body.contains("could not commit"), "{body}");
17696        assert!(!body.contains("could not run"), "{body}");
17697        assert!(
17698            body.contains("The fix is committed on the branch"),
17699            "{body}"
17700        );
17701        assert!(!body.contains("sandbox"), "{body}");
17702        assert!(
17703            body.contains("Tests (`verify.e2e`): passed on the final commit"),
17704            "{body}"
17705        );
17706        assert!(
17707            body.contains("Gate (`verify.gate`): passed on the final commit"),
17708            "{body}"
17709        );
17710        // The raw report is untouched in the record.
17711        assert!(
17712            state.reviews[0]
17713                .fix
17714                .as_ref()
17715                .unwrap()
17716                .notes
17717                .contains("could not commit")
17718        );
17719    }
17720
17721    #[test]
17722    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17723        let mut state = review_state(&["feat: x"]);
17724        let mut r1 = round_with_notes(1, Some(SANDBOXED));
17725        r1.fix.as_mut().unwrap().committed = false;
17726        let mut r2 = passing_round(2, Some(FINAL));
17727        r2.e2e[0].code = Some(1);
17728        state.reviews = vec![r1, r2];
17729        let body = body_at(&state);
17730        assert!(!body.contains("could not commit"), "{body}");
17731        assert!(!body.contains("could not run"), "{body}");
17732        assert!(body.contains("This round produced no commit"), "{body}");
17733        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17734        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17735        assert!(!body.contains("passed"), "{body}");
17736    }
17737
17738    #[test]
17739    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17740        let mut state = review_state(&["feat: x"]);
17741        state.reviews = vec![
17742            round_with_notes(1, Some(SANDBOXED)),
17743            passing_round(2, Some("0123456789abcdef")),
17744        ];
17745        ok_gate(&mut state, Some("0123456789abcdef"));
17746        let body = body_at(&state);
17747        assert!(!body.contains("HTTP tests could not run"), "{body}");
17748        // The commit claim is still settled by the structured record.
17749        assert!(!body.contains("could not commit"), "{body}");
17750        assert!(
17751            body.contains("passed on an earlier commit (`0123456`)"),
17752            "{body}"
17753        );
17754        assert!(!body.contains("passed on the final commit"), "{body}");
17755
17756        // A gate recorded before `gate_head` existed is "not recorded".
17757        ok_gate(&mut state, None);
17758        let body = body_at(&state);
17759        assert!(
17760            body.contains("passed on a commit that was not recorded"),
17761            "{body}"
17762        );
17763    }
17764
17765    #[test]
17766    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17767        let state = state_with_summary(
17768            "add retries to the client",
17769            "TITLE: feat: retries\n\nDid it.",
17770        );
17771        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17772        assert_eq!(m.title, "feat: retries");
17773        assert!(
17774            m.body.contains("<summary>Original task</summary>"),
17775            "{}",
17776            m.body
17777        );
17778        assert!(!m.body.contains("Commits under review"), "{}", m.body);
17779        assert!(
17780            m.body
17781                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17782            "{}",
17783            m.body
17784        );
17785    }
17786
17787    #[test]
17788    fn pr_message_review_bounds_a_long_english_subject() {
17789        let long = format!("feat: {}", "word ".repeat(100));
17790        let m = pr_message(&review_state(&[&long]), 'A');
17791        assert!(m.title.starts_with("feat: word"), "{}", m.title);
17792        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17793    }
17794
17795    #[test]
17796    fn pr_message_implementation_run_is_unchanged_by_review_support() {
17797        let state = state_with_summary("add retries\n\ndetails", "- did some things");
17798        let m = pr_message(&state, 'A');
17799        assert_eq!(m.title, "add retries");
17800        assert!(m.body.contains("<summary>Original task</summary>"));
17801        assert!(!m.body.contains("Commits under review"));
17802        assert_eq!(landing_subject_source(&state), state.instruction);
17803    }
17804
17805    #[test]
17806    fn review_run_squash_subject_is_the_change_not_the_prompt() {
17807        let state = review_state(&["feat: the change", "fix: typo"]);
17808        let source = landing_subject_source(&state);
17809        assert_eq!(land::merge_subject("", &source), "feat: the change");
17810        assert_eq!(
17811            land::merge_subject("magi: candidate A (uncommitted work)", &source),
17812            "feat: the change"
17813        );
17814        // An operator's rename still wins.
17815        assert_eq!(
17816            land::merge_subject("feat: renamed by hand", &source),
17817            "feat: renamed by hand"
17818        );
17819        let blank = review_state(&["日本語"]);
17820        assert!(
17821            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17822        );
17823    }
17824
17825    #[test]
17826    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17827        let state = review_state(&["feat: the change"]);
17828        let source = landing_subject_source(&state);
17829        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17830        assert_eq!(
17831            land::merge_subject(landing_title(&state, old), &source),
17832            "feat: the change"
17833        );
17834        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17835        let task = state_with_summary("add retries", "");
17836        assert_eq!(landing_title(&task, old), old);
17837    }
17838
17839    #[test]
17840    fn pr_message_describes_the_change_not_the_task() {
17841        let state = state_with_summary(
17842            "今回やってほしいこと: results projector を直す",
17843            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17844        );
17845        let m = pr_message(&state, 'A');
17846        assert_eq!(m.title, "fix(web): batch the runs list reads");
17847        assert!(
17848            m.body.starts_with("## Summary\n\n- reads run.json once"),
17849            "{}",
17850            m.body
17851        );
17852        assert!(!m.body.contains("TITLE:"), "{}", m.body);
17853        let task_at = m.body.find("今回やってほしいこと").unwrap();
17854        let details_at = m.body.find("<details>").unwrap();
17855        assert!(
17856            details_at < task_at,
17857            "the task lives inside <details>: {}",
17858            m.body
17859        );
17860        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17861        assert!(m.body.contains("magi:candidate-a"));
17862    }
17863
17864    #[test]
17865    fn pr_message_falls_back_to_the_task_without_a_title_line() {
17866        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17867        let m = pr_message(&state, 'A');
17868        assert_eq!(m.title, "add retries");
17869        assert!(
17870            m.body.contains("## Summary\n\n- did some things"),
17871            "{}",
17872            m.body
17873        );
17874
17875        let none = RunState::new(
17876            PathBuf::from("/repo"),
17877            "main".to_owned(),
17878            "abc1234".to_owned(),
17879            "add retries".to_owned(),
17880            Config::default(),
17881        );
17882        let m = pr_message(&none, 'A');
17883        assert_eq!(m.title, "add retries");
17884        assert!(!m.body.contains("## Summary"), "{}", m.body);
17885    }
17886
17887    #[test]
17888    fn pr_message_refuses_the_candidate_commit_subject() {
17889        for bad in [
17890            "TITLE: magi: candidate A (uncommitted work)",
17891            "TITLE: chore: stuff (uncommitted work)",
17892            "TITLE:   ",
17893        ] {
17894            let state = state_with_summary("add retries", bad);
17895            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17896        }
17897    }
17898
17899    #[test]
17900    fn pr_message_bounds_a_very_long_task_and_title() {
17901        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17902        let state = state_with_summary(&long, "- nothing");
17903        let m = pr_message(&state, 'A');
17904        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17905        assert!(!m.title.contains('\n'));
17906
17907        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17908        let m = pr_message(&state, 'A');
17909        assert!(m.title.starts_with("feat: "));
17910        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17911        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17912    }
17913
17914    fn long_title_of(instruction: &str) -> String {
17915        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17916    }
17917
17918    #[test]
17919    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17920        let first = "Make the landing path keep a readable title for long tasks";
17921        let line = format!(
17922            "{first}. {}",
17923            "And then keep going with more words ".repeat(20)
17924        );
17925        let t = long_title_of(&line);
17926        assert_eq!(t, first);
17927        assert!(!t.starts_with("chore: land"));
17928    }
17929
17930    #[test]
17931    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17932        let line = "word ".repeat(200);
17933        let t = long_title_of(&line);
17934        assert!(t.ends_with("word..."), "{t}");
17935        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17936    }
17937
17938    #[test]
17939    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17940        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17941            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17942        }
17943    }
17944
17945    #[test]
17946    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17947        let mut state = state_with_summary("再試行を追加する", "- nothing");
17948        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17949        state.config.graph.github_language = "ja".into();
17950        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17951    }
17952
17953    #[test]
17954    fn pr_message_title_limit_is_exact() {
17955        let at = "a".repeat(PR_TITLE_MAX);
17956        assert_eq!(long_title_of(&at), at);
17957        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17958        assert!(over.ends_with("..."), "{over}");
17959        assert_eq!(over.chars().count(), PR_TITLE_MAX);
17960    }
17961
17962    #[test]
17963    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17964        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17965        let t = long_title_of(&line);
17966        assert!(t.ends_with("..."), "{t}");
17967        assert!(t.is_ascii(), "{t}");
17968    }
17969
17970    #[test]
17971    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17972        let line = format!(
17973            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17974            "plus more filler words ".repeat(20)
17975        );
17976        let t = long_title_of(&line);
17977        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17978    }
17979
17980    #[test]
17981    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17982        let line = format!("\n\n# {}", "title words ".repeat(40));
17983        let state = state_with_summary(&line, "- nothing");
17984        let m = pr_message(&state, 'A');
17985        assert!(m.title.starts_with("title words"), "{}", m.title);
17986        assert_eq!(
17987            land::merge_subject(&m.title, &landing_subject_source(&state)),
17988            m.title
17989        );
17990        // An operator's rename wins untouched.
17991        assert_eq!(
17992            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17993            "feat: renamed by hand"
17994        );
17995    }
17996
17997    #[tokio::test]
17998    async fn github_text_rewrite_is_bounded_and_falls_back() {
17999        crate::run::pin_test_home();
18000        for (reply, accepted) in [
18001            (
18002                "TITLE: fix: retries\nAdd retries for failed requests.",
18003                true,
18004            ),
18005            (
18006                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18007                false,
18008            ),
18009            (
18010                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18011                true,
18012            ),
18013            (
18014                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18015                true,
18016            ),
18017            (
18018                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18019                true,
18020            ),
18021            (
18022                "TITLE: fix: retries\n日本語の説明です。 token=secret",
18023                false,
18024            ),
18025            ("TITLE: fix: retries\n", false),
18026        ] {
18027            let dir = tempfile::tempdir().unwrap();
18028            let mut runner = runner_at(RunStatus::Gating);
18029            runner.state = state_with_summary(
18030                "add retries",
18031                "TITLE: fix: retries\n日本語の説明を書きます。",
18032            );
18033            runner.state.repo = dir.path().to_owned();
18034            runner.state.candidates[0].worktree = dir.path().to_owned();
18035            let mut author = spec("alpha");
18036            author.command = vec![
18037                "sh".into(),
18038                "-c".into(),
18039                "printf '%s' \"$REWRITE_REPLY\"".into(),
18040            ];
18041            author.env.insert("REWRITE_REPLY".into(), reply.into());
18042            runner.state.config.agents = vec![author];
18043            seed_public_summary(&mut runner);
18044            let winner = runner.state.candidates[0].clone();
18045            let first = runner
18046                .guarded_pr_message(&winner, None, true)
18047                .await
18048                .unwrap();
18049            let message = if accepted {
18050                first.expect("a passing rewrite never asks")
18051            } else {
18052                // Withheld: the run parks on a question and posts nothing yet.
18053                assert!(first.is_none());
18054                assert!(runner.state.parked);
18055                let store = ask::Questions::open();
18056                let asked: Vec<_> = store
18057                    .list()
18058                    .into_iter()
18059                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18060                    .collect();
18061                assert_eq!(asked.len(), 1);
18062                assert!(!asked[0].detail.contains("secret"));
18063                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18064                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18065                assert_eq!(
18066                    asked[0]
18067                        .choices
18068                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
18069                    rewrite.is_some()
18070                );
18071                // Silence: the question lapses and the neutral text is used,
18072                // without a second rewrite call or a second question.
18073                store
18074                    .update(&asked[0].id, |q| {
18075                        q.abandon("test");
18076                        Ok(())
18077                    })
18078                    .unwrap();
18079                let again = runner
18080                    .guarded_pr_message(&winner, None, true)
18081                    .await
18082                    .unwrap()
18083                    .expect("an abandoned question falls back");
18084                let n = store
18085                    .list()
18086                    .into_iter()
18087                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18088                    .count();
18089                assert_eq!(n, 1, "asked once per rejected text");
18090                again
18091            };
18092            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18093            assert!(!message.body.contains("secret"));
18094            assert!(!message.body.contains("prefix"));
18095            assert!(!message.title.contains("secret"));
18096            assert_eq!(
18097                message.body.contains("Add retries for failed requests."),
18098                accepted
18099            );
18100            assert_eq!(
18101                runner.state.seats["github-text-rewrite"].turns, 1,
18102                "only one rewrite invocation"
18103            );
18104            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18105            if accepted {
18106                assert!(runner.state.github_text.is_none());
18107                assert!(!runner.state.parked);
18108                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18109                assert!(message.body.contains("## Final verification"));
18110                assert!(message.body.contains("Evidence: final diff"));
18111                assert!(
18112                    message
18113                        .body
18114                        .contains(&format!("magi:run/{}", runner.state.id))
18115                );
18116            }
18117            if !accepted {
18118                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18119                assert!(
18120                    message
18121                        .body
18122                        .contains(&format!("magi:run/{}", runner.state.id))
18123                );
18124            }
18125        }
18126    }
18127
18128    fn scripted(id: &str, reply: &str) -> AgentSpec {
18129        let mut a = spec(id);
18130        a.command = vec![
18131            "sh".into(),
18132            "-c".into(),
18133            "printf '%s' \"$REWRITE_REPLY\"".into(),
18134        ];
18135        a.env.insert("REWRITE_REPLY".into(), reply.into());
18136        a
18137    }
18138
18139    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18140    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18141
18142    /// Gate tests inject generated public output, not internal reports.
18143    fn seed_public_summary(runner: &mut Runner) {
18144        let repo = &runner.state.repo;
18145        let g = |args: &[&str]| {
18146            let out = std::process::Command::new("git")
18147                .args(args)
18148                .current_dir(repo)
18149                .quiet()
18150                .output()
18151                .unwrap();
18152            assert!(out.status.success(), "git {args:?}");
18153            String::from_utf8_lossy(&out.stdout).trim().to_owned()
18154        };
18155        g(&["init", "-q", "-b", "main"]);
18156        g(&[
18157            "-c",
18158            "user.name=Test",
18159            "-c",
18160            "user.email=test@example.invalid",
18161            "commit",
18162            "-q",
18163            "--allow-empty",
18164            "-m",
18165            "feat: fixture",
18166        ]);
18167        let head = g(&["rev-parse", "HEAD"]);
18168        g(&["update-ref", "refs/remotes/origin/main", &head]);
18169        let evidence = crate::pr_outcome::summary_evidence(
18170            &runner.state,
18171            crate::pr_outcome::FinalHead::At(&head),
18172        );
18173        let generated = &runner.state.candidates[0].summary;
18174        runner.state.change_summary = Some(crate::run::ChangeSummary {
18175            base: head.clone(),
18176            head,
18177            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18178            title: summary_title(generated).unwrap(),
18179            body: summary_without_title(generated),
18180        });
18181    }
18182
18183    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18184        let mut runner = runner_at(RunStatus::Gating);
18185        runner.state = state_with_summary("add retries", JA_SUMMARY);
18186        if review_only {
18187            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18188        }
18189        runner.state.repo = dir.to_owned();
18190        runner.state.candidates[0].worktree = dir.to_owned();
18191        runner.state.config.agents = agents;
18192        seed_public_summary(&mut runner);
18193        runner
18194    }
18195
18196    fn gate_questions(runner: &Runner) -> usize {
18197        ask::Questions::open()
18198            .list()
18199            .into_iter()
18200            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18201            .count()
18202    }
18203
18204    #[tokio::test]
18205    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18206        crate::run::pin_test_home();
18207        let dir = tempfile::tempdir().unwrap();
18208        let mut runner = rewrite_runner(
18209            dir.path(),
18210            true,
18211            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18212        );
18213        runner.state.config.roles.language_judge =
18214            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18215        let winner = runner.state.candidates[0].clone();
18216        let msg = runner
18217            .guarded_pr_message(&winner, None, true)
18218            .await
18219            .unwrap()
18220            .expect("an English rewrite posts without asking");
18221        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18222        assert!(msg.body.contains("<details>kept</details>"));
18223        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18224        assert_eq!(gate_questions(&runner), 0);
18225        assert!(runner.state.github_text.is_none());
18226        assert!(!runner.state.seats.contains_key("impl-A"));
18227        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18228    }
18229
18230    #[tokio::test]
18231    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18232        crate::run::pin_test_home();
18233        let dir = tempfile::tempdir().unwrap();
18234        let mut runner = rewrite_runner(
18235            dir.path(),
18236            false,
18237            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18238        );
18239        runner.state.config.roles.language_judge =
18240            Some(crate::config::AgentChoice::One("beta".into()));
18241        runner.state.seats.insert(
18242            "impl-A".into(),
18243            SeatState::new("impl-A", "alpha", runner.state.seed),
18244        );
18245        let winner = runner.state.candidates[0].clone();
18246        let msg = runner
18247            .guarded_pr_message(&winner, None, true)
18248            .await
18249            .unwrap()
18250            .expect("rewritten");
18251        assert!(msg.body.contains("<details>kept</details>"));
18252        assert_eq!(runner.state.seats["impl-A"].turns, 0);
18253        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18254        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18255    }
18256
18257    #[tokio::test]
18258    async fn rewrite_reuses_a_resumable_implementer_session() {
18259        crate::run::pin_test_home();
18260        let dir = tempfile::tempdir().unwrap();
18261        let mut runner = rewrite_runner(
18262            dir.path(),
18263            false,
18264            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18265        );
18266        runner.state.config.roles.language_judge =
18267            Some(crate::config::AgentChoice::One("beta".into()));
18268        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18269        seat.turns = 1;
18270        runner.state.seats.insert("impl-A".into(), seat);
18271        let winner = runner.state.candidates[0].clone();
18272        runner
18273            .guarded_pr_message(&winner, None, true)
18274            .await
18275            .unwrap()
18276            .expect("rewritten");
18277        assert_eq!(runner.state.seats["impl-A"].turns, 2);
18278        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18279    }
18280
18281    #[tokio::test]
18282    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
18283        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
18284        // session, was out of quota, and nothing else was asked.
18285        crate::run::pin_test_home();
18286        let dir = tempfile::tempdir().unwrap();
18287        let mut exhausted = spec("alpha");
18288        exhausted.command = vec!["false".into()];
18289        let mut runner = rewrite_runner(
18290            dir.path(),
18291            false,
18292            vec![exhausted, scripted("beta", EN_REWRITE)],
18293        );
18294        // The implementer's own agent is in the role too: it is not asked twice.
18295        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18296            "alpha".into(),
18297            "beta".into(),
18298        ]));
18299        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18300        seat.turns = 1;
18301        runner.state.seats.insert("impl-A".into(), seat);
18302        let winner = runner.state.candidates[0].clone();
18303        let msg = runner
18304            .guarded_pr_message(&winner, None, true)
18305            .await
18306            .unwrap()
18307            .expect("the role rewrites after the implementer fails");
18308        assert!(msg.body.contains("`src/net.rs`"));
18309        assert_eq!(
18310            runner.state.seats["impl-A"].turns, 2,
18311            "the implementer was tried once"
18312        );
18313        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18314        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18315    }
18316
18317    #[tokio::test]
18318    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18319        crate::run::pin_test_home();
18320        let dir = tempfile::tempdir().unwrap();
18321        let mut broken = spec("bad");
18322        broken.command = vec!["false".into()];
18323        let mut runner = rewrite_runner(
18324            dir.path(),
18325            true,
18326            vec![
18327                broken,
18328                scripted("noformat", "Add retries."),
18329                scripted("good", EN_REWRITE),
18330            ],
18331        );
18332        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18333            "bad".into(),
18334            "noformat".into(),
18335            "good".into(),
18336        ]));
18337        let winner = runner.state.candidates[0].clone();
18338        let msg = runner
18339            .guarded_pr_message(&winner, None, true)
18340            .await
18341            .unwrap()
18342            .expect("the third agent rewrites");
18343        assert!(msg.body.contains("`src/net.rs`"));
18344        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18345        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18346    }
18347
18348    #[tokio::test]
18349    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18350        crate::run::pin_test_home();
18351        // (agents, role): still Japanese; no TITLE; nothing resolvable.
18352        for (agents, role, saved) in [
18353            (
18354                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18355                Some("beta"),
18356                true,
18357            ),
18358            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18359            (Vec::new(), None, false),
18360        ] {
18361            let dir = tempfile::tempdir().unwrap();
18362            let mut runner = rewrite_runner(dir.path(), true, agents);
18363            runner.state.config.roles.language_judge =
18364                role.map(|r| crate::config::AgentChoice::One(r.into()));
18365            let winner = runner.state.candidates[0].clone();
18366            let first = runner
18367                .guarded_pr_message(&winner, None, true)
18368                .await
18369                .unwrap();
18370            assert!(first.is_none(), "withheld text parks the run");
18371            assert!(runner.state.parked);
18372            assert_eq!(gate_questions(&runner), 1);
18373            let g = runner.state.github_text.as_ref().unwrap();
18374            assert_eq!(g.rewrite.is_some(), saved);
18375            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18376        }
18377    }
18378
18379    #[tokio::test]
18380    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18381        crate::run::pin_test_home();
18382        let dir = tempfile::tempdir().unwrap();
18383        let mut runner = runner_at(RunStatus::Gating);
18384        runner.state =
18385            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18386        runner.state.repo = dir.path().to_owned();
18387        runner.state.candidates[0].worktree = dir.path().to_owned();
18388        let mut author = spec("alpha");
18389        author.command = vec![
18390            "sh".into(),
18391            "-c".into(),
18392            "printf '%s' \"$REWRITE_REPLY\"".into(),
18393        ];
18394        author.env.insert(
18395            "REWRITE_REPLY".into(),
18396            format!(
18397                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18398                "English sentence. ".repeat(600)
18399            ),
18400        );
18401        runner.state.config.agents = vec![author];
18402        seed_public_summary(&mut runner);
18403        let winner = runner.state.candidates[0].clone();
18404        assert!(
18405            runner
18406                .guarded_pr_message(&winner, None, true)
18407                .await
18408                .unwrap()
18409                .is_none()
18410        );
18411        let record = runner.state.github_text.clone().unwrap();
18412        let rewrite = record.rewrite.clone().unwrap();
18413        assert_eq!(rewrite.failed, ["body-language"]);
18414        assert!(
18415            rewrite
18416                .body
18417                .ends_with(&format!("magi:run/{}", runner.state.id))
18418        );
18419        let store = ask::Questions::open();
18420        let q = store.get(record.question.as_ref().unwrap()).unwrap();
18421        assert!(
18422            q.choices
18423                .contains(&crate::github_text::USE_REWRITE.to_owned())
18424        );
18425        for label in [
18426            "ORIGINAL",
18427            "FALLBACK",
18428            "REWRITE",
18429            "Rules still failed: body-language",
18430            "truncated",
18431        ] {
18432            assert!(q.detail.contains(label), "{label}");
18433        }
18434        let artifact = std::fs::read_to_string(crate::run::artifact_path(
18435            &runner.state,
18436            &crate::github_text::artifact_name(&record.fingerprint),
18437        ))
18438        .unwrap();
18439        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18440        // Restart from serialized state, then retry after a bad owner title.
18441        runner.state =
18442            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18443        store
18444            .update(&q.id, |q| {
18445                q.say("まだ日本語")?;
18446                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18447            })
18448            .unwrap();
18449        assert!(
18450            runner
18451                .guarded_pr_message(&winner, None, true)
18452                .await
18453                .unwrap()
18454                .is_none()
18455        );
18456        let retry = runner.state.github_text.as_ref().unwrap();
18457        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18458        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18459        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18460        assert!(
18461            q.posting_comparison
18462                .as_ref()
18463                .unwrap()
18464                .matches_detail(&q.detail)
18465        );
18466        store
18467            .update(&q.id, |q| {
18468                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18469            })
18470            .unwrap();
18471        let message = runner
18472            .guarded_pr_message(&winner, None, true)
18473            .await
18474            .unwrap()
18475            .unwrap();
18476        assert_eq!(message.title, rewrite.title);
18477        assert_eq!(message.body, rewrite.body);
18478        assert_eq!(
18479            runner.state.github_text.as_ref().unwrap().choice,
18480            Some(GithubTextChoice::Rewrite)
18481        );
18482        let again = runner
18483            .guarded_pr_message(&winner, None, true)
18484            .await
18485            .unwrap()
18486            .unwrap();
18487        assert_eq!(again.title, message.title);
18488        assert_eq!(again.body, message.body);
18489        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18490        assert!(
18491            !runner
18492                .state
18493                .events
18494                .iter()
18495                .any(|e| e.message.contains(&rewrite.title)
18496                    || e.message.contains("English sentence."))
18497        );
18498    }
18499
18500    #[tokio::test]
18501    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18502        crate::run::pin_test_home();
18503        let dir = tempfile::tempdir().unwrap();
18504        let mut runner = runner_at(RunStatus::Gating);
18505        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18506        runner.state.repo = dir.path().to_owned();
18507        runner.state.candidates[0].worktree = dir.path().to_owned();
18508        seed_public_summary(&mut runner);
18509        let winner = runner.state.candidates[0].clone();
18510        assert!(
18511            runner
18512                .guarded_pr_message(&winner, None, true)
18513                .await
18514                .unwrap()
18515                .is_none()
18516        );
18517        let record = runner.state.github_text.as_ref().unwrap();
18518        assert!(record.rewrite.is_none());
18519        // Records written before the additive field deserialize with no rewrite.
18520        let mut old = serde_json::to_value(record).unwrap();
18521        old.as_object_mut().unwrap().remove("rewrite");
18522        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18523        assert!(old.rewrite.is_none());
18524        let id = record.question.clone().unwrap();
18525        ask::Questions::open()
18526            .update(&id, |q| {
18527                q.choices.push(crate::github_text::USE_REWRITE.into());
18528                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18529            })
18530            .unwrap();
18531        let message = runner
18532            .guarded_pr_message(&winner, None, true)
18533            .await
18534            .unwrap()
18535            .unwrap();
18536        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18537        assert_eq!(
18538            runner.state.github_text.as_ref().unwrap().choice,
18539            Some(GithubTextChoice::Fallback)
18540        );
18541    }
18542
18543    #[tokio::test]
18544    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18545        crate::run::pin_test_home();
18546        let dir = tempfile::tempdir().unwrap();
18547        let mut runner = runner_at(RunStatus::Gating);
18548        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18549        runner.state.repo = dir.path().to_owned();
18550        runner.state.candidates[0].worktree = dir.path().to_owned();
18551        seed_public_summary(&mut runner);
18552        let winner = runner.state.candidates[0].clone();
18553        assert!(
18554            runner
18555                .guarded_pr_message(&winner, None, true)
18556                .await
18557                .unwrap()
18558                .is_none()
18559        );
18560        let store = ask::Questions::open();
18561        let find = |run: &str| {
18562            store
18563                .list()
18564                .into_iter()
18565                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18566                .unwrap()
18567        };
18568        let run = runner.state.id.clone();
18569        let q = find(&run);
18570        let fp = runner
18571            .state
18572            .github_text
18573            .as_ref()
18574            .unwrap()
18575            .fingerprint
18576            .clone();
18577        let name = crate::github_text::artifact_name(&fp);
18578        let path = crate::run::artifact_path(&runner.state, &name);
18579        assert!(q.detail.contains(&path.display().to_string()));
18580        assert!(q.detail.contains("title-language: title"));
18581        let saved = std::fs::read_to_string(&path).expect("artifact written");
18582        assert!(saved.contains("日本語のタイトル"));
18583        assert!(q.cwd.is_none());
18584        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18585        // A bad replacement is refused and asked about once more.
18586        let say = |id: &str, text: &str, choice: &str| {
18587            store
18588                .update(id, |q| {
18589                    q.thread.push(ask::Turn {
18590                        who: ask::Who::Operator,
18591                        body: text.to_owned(),
18592                        at: jiff::Timestamp::now(),
18593                        note: None,
18594                    });
18595                    q.answer(ask::Answer::Choice(choice.to_owned()))
18596                })
18597                .unwrap();
18598        };
18599        say(&q.id, "まだ日本語", "use my text");
18600        assert!(
18601            runner
18602                .guarded_pr_message(&winner, None, true)
18603                .await
18604                .unwrap()
18605                .is_none()
18606        );
18607        let second = find(&run);
18608        assert_ne!(second.id, q.id);
18609        assert!(second.detail.contains("did not pass"));
18610        // The retry still compares the original text; the rejected
18611        // replacement is named by category only.
18612        assert!(second.detail.contains("日本語のタイトル"));
18613        assert!(!second.detail.contains("まだ日本語"));
18614        assert!(second.detail.contains(&path.display().to_string()));
18615        say(&second.id, "fix: retry failed requests", "use my text");
18616        let message = runner
18617            .guarded_pr_message(&winner, None, true)
18618            .await
18619            .unwrap()
18620            .expect("a vetted replacement is posted");
18621        assert_eq!(message.title, "fix: retry failed requests");
18622        assert_eq!(
18623            runner
18624                .state
18625                .github_text
18626                .as_ref()
18627                .unwrap()
18628                .chosen_title
18629                .as_deref(),
18630            Some("fix: retry failed requests")
18631        );
18632        assert!(runner.state.github_text.as_ref().unwrap().resolved);
18633        // Resumed again: the saved decision is applied, nothing is re-asked.
18634        let again = runner
18635            .guarded_pr_message(&winner, None, true)
18636            .await
18637            .unwrap()
18638            .unwrap();
18639        assert_eq!(again.title, message.title);
18640        assert_eq!(
18641            store
18642                .list()
18643                .into_iter()
18644                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18645                .count(),
18646            2
18647        );
18648    }
18649
18650    #[test]
18651    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18652        // What magi itself writes stays English under any configured language,
18653        // so a future localisation of these headings fails here. (The agents'
18654        // own text is also checked by the posting gate.)
18655        let mut state = state_with_summary(
18656            "add retries",
18657            "TITLE: fix(web): batch reads\n- reads run.json once",
18658        );
18659        state.config.graph.language = "ja".to_owned();
18660        let m = pr_message(&state, 'A');
18661        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18662        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18663
18664        // The task is the operator's own text: it goes in untouched, and the
18665        // fallback title (no summary) may be in its language too.
18666        let task = "今回やってほしいこと: results projector を直す";
18667        let mut state = state_with_summary(task, "- no title line");
18668        state.config.graph.language = "ja".to_owned();
18669        let m = pr_message(&state, 'A');
18670        assert_eq!(
18671            m.title,
18672            format!("chore: land candidate A of run {}", state.id)
18673        );
18674        assert!(
18675            m.body.contains(&format!(
18676                "<summary>Original task</summary>\n\n{task}\n\n</details>"
18677            )),
18678            "{}",
18679            m.body
18680        );
18681    }
18682
18683    #[test]
18684    fn pr_message_scrubs_home_paths_and_addresses() {
18685        let state = state_with_summary(
18686            "fix it in /Users/someone/src/x",
18687            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18688        );
18689        let m = pr_message(&state, 'A');
18690        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18691            assert!(!m.body.contains(leak), "{}", m.body);
18692        }
18693        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18694    }
18695
18696    #[test]
18697    fn pr_message_survives_a_task_that_closes_details() {
18698        let state = state_with_summary("a </details> b", "TITLE: fix: x");
18699        let m = pr_message(&state, 'A');
18700        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18701    }
18702
18703    #[test]
18704    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18705        let cmd = manual_merge_command(
18706            MergeStyle::Squash,
18707            Path::new("/repo"),
18708            "b",
18709            "fix: \"quoted\" $(x) `y`\n\nbody",
18710        );
18711        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18712    }
18713
18714    #[test]
18715    fn manual_merge_command_matches_the_configured_style() {
18716        let repo = Path::new("/repo");
18717        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18718
18719        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18720        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18721
18722        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18723        assert_eq!(
18724            squash,
18725            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18726             \"Merge magi run 0832 (candidate A)\""
18727        );
18728
18729        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18730        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18731    }
18732
18733    #[test]
18734    fn a_nudge_gets_a_quarter_of_the_budget() {
18735        // The judge and implement budgets magi ships with.
18736        assert_eq!(retry_budget(secs(1200), true), secs(300));
18737        assert_eq!(retry_budget(secs(3600), true), secs(900));
18738    }
18739
18740    #[test]
18741    fn a_resent_prompt_keeps_the_whole_budget() {
18742        // The seat kept no context, so the retry is the original job again and
18743        // shortening it would only guarantee a second failure.
18744        assert_eq!(retry_budget(secs(1200), false), secs(1200));
18745        assert_eq!(retry_budget(secs(60), false), secs(60));
18746    }
18747
18748    #[test]
18749    fn the_floor_never_exceeds_the_original_budget() {
18750        // A short configured timeout must not be *raised* by the floor: the
18751        // operator asked for a bound, and a retry may not outlast the attempt
18752        // it is retrying.
18753        assert_eq!(retry_budget(secs(60), true), secs(60));
18754        assert_eq!(retry_budget(secs(480), true), secs(120));
18755        assert_eq!(retry_budget(secs(0), true), secs(0));
18756    }
18757
18758    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18759        agent::CommandEvidence {
18760            id: "item1".to_owned(),
18761            description: "cargo test".to_owned(),
18762            exit_code,
18763            result_summary: String::new(),
18764            source: "codex".to_owned(),
18765        }
18766    }
18767
18768    #[test]
18769    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18770        // No evidence is not the same fact as unconfirmed evidence: a
18771        // backend with no adapter, or a reply that ran no commands at all,
18772        // must not be misread as carrying a dangling job.
18773        assert!(!has_unconfirmed_command(&[]));
18774    }
18775
18776    #[test]
18777    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18778        // Deliberately not a check on the exit code's *value*: a fixer
18779        // legitimately runs something that fails mid-iteration before it
18780        // succeeds, and that must never by itself reopen a valid report.
18781        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18782        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18783        assert!(!has_unconfirmed_command(&[
18784            evidence(Some(0)),
18785            evidence(Some(101))
18786        ]));
18787    }
18788
18789    #[test]
18790    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18791        assert!(has_unconfirmed_command(&[
18792            evidence(Some(0)),
18793            evidence(None)
18794        ]));
18795    }
18796
18797    #[test]
18798    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18799        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18800        assert_eq!(
18801            verified_noop_claim(true, &[], text).as_deref(),
18802            Some("already fixed by b32cfc4, on main.")
18803        );
18804    }
18805
18806    #[test]
18807    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18808        // A timeout or a bad exit code reads as the ordinary loss it is,
18809        // whatever the reply's own prose claims.
18810        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18811        assert!(verified_noop_claim(false, &[], text).is_none());
18812    }
18813
18814    #[test]
18815    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18816        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18817        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18818        // A confirmed command alongside the marker is fine.
18819        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18820    }
18821
18822    #[test]
18823    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18824        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18825    }
18826
18827    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
18828    /// pair, in order, labelled A, B, C, ...
18829    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18830        runner.state.candidates = shape
18831            .iter()
18832            .enumerate()
18833            .map(|(i, &(empty, verified))| Candidate {
18834                index: i,
18835                label: (b'A' + i as u8) as char,
18836                agent: "sonnet".to_owned(),
18837                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18838                worktree: PathBuf::from(format!("/wt/{i}")),
18839                summary: String::new(),
18840                stat: String::new(),
18841                files: 0,
18842                commits: 0,
18843                empty,
18844                failed: None,
18845                verified_noop: verified.map(str::to_owned),
18846                duration_ms: 0,
18847                folded: false,
18848            })
18849            .collect();
18850    }
18851
18852    #[test]
18853    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18854        ask_test_home();
18855        let mut runner = runner_at(RunStatus::Implementing);
18856        set_candidates(
18857            &mut runner,
18858            &[
18859                (true, Some("already on main at b32cfc4")),
18860                (true, Some("same fix, see the existing test")),
18861            ],
18862        );
18863
18864        runner
18865            .after_implement()
18866            .expect("a verified no-op is not an error");
18867
18868        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18869    }
18870
18871    #[test]
18872    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18873        ask_test_home();
18874        let mut runner = runner_at(RunStatus::Implementing);
18875        // Candidate A declares a verified no-op; candidate B simply wrote
18876        // nothing and said nothing about why. One candidate's claim is not
18877        // the whole run's agreement.
18878        set_candidates(
18879            &mut runner,
18880            &[(true, Some("already on main at b32cfc4")), (true, None)],
18881        );
18882
18883        let err = runner
18884            .after_implement()
18885            .expect_err("an unverified empty candidate must still fail the run");
18886
18887        assert!(
18888            err.to_string().contains("no candidate produced a change"),
18889            "{err}"
18890        );
18891        assert_eq!(runner.state.status, RunStatus::Failed);
18892    }
18893
18894    #[test]
18895    fn after_implement_still_fails_an_ordinary_all_empty_run() {
18896        ask_test_home();
18897        let mut runner = runner_at(RunStatus::Implementing);
18898        set_candidates(&mut runner, &[(true, None), (true, None)]);
18899
18900        let err = runner
18901            .after_implement()
18902            .expect_err("no candidate declared anything; this is an ordinary failure");
18903
18904        assert!(
18905            err.to_string().contains("no candidate produced a change"),
18906            "{err}"
18907        );
18908        assert_eq!(runner.state.status, RunStatus::Failed);
18909    }
18910}
18911
18912/// A graft rollback whose restore could not be confirmed (see
18913/// [`Runner::graft`]).
18914#[derive(Debug)]
18915struct GraftRestoreFailed(String);
18916
18917impl std::fmt::Display for GraftRestoreFailed {
18918    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18919        write!(
18920            f,
18921            "could not restore the winner to its pre-graft commit: {}",
18922            self.0
18923        )
18924    }
18925}
18926
18927impl std::error::Error for GraftRestoreFailed {}
18928
18929/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
18930/// not the agent: they never draw on an agent's own timeout, and running out of
18931/// them never reads as an agent failure.
18932#[cfg(test)]
18933mod resource_wait_tests {
18934    use super::*;
18935    use std::collections::BTreeMap;
18936
18937    fn sh(id: &str, script: &str) -> AgentSpec {
18938        AgentSpec {
18939            id: id.to_owned(),
18940            kind: crate::config::AgentKind::Command,
18941            model: None,
18942            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18943            extra_args: Vec::new(),
18944            env: BTreeMap::new(),
18945            prompt_delivery: None,
18946            max_turn_secs: None,
18947        }
18948    }
18949
18950    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18951        SeatJob {
18952            seat: SeatState::new("impl-A", &spec.id, 1),
18953            spec,
18954            cwd: dir.to_path_buf(),
18955            prompt: "do it".to_owned(),
18956            timeout,
18957            allow_write: true,
18958            sessions: false,
18959            artifacts: dir.join("art"),
18960            stem: "impl-A".to_owned(),
18961            handover: None,
18962        }
18963    }
18964
18965    fn state(resource_wait_secs: u64) -> RunState {
18966        let mut config = Config::default();
18967        config.graph.timeout_resource_wait = resource_wait_secs;
18968        RunState::new(
18969            PathBuf::from("/repo"),
18970            "main".to_owned(),
18971            "abc1234".to_owned(),
18972            "task".to_owned(),
18973            config,
18974        )
18975    }
18976
18977    fn hold(cache: &Path) -> crate::cache::Guard {
18978        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18979        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18980            crate::cache::AcquireOutcome::Acquired(g) => g,
18981            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18982        }
18983    }
18984
18985    async fn run(
18986        jobs: Vec<SeatJob>,
18987        sem: Arc<Semaphore>,
18988        cache: Option<&Path>,
18989        st: &mut RunState,
18990    ) -> Vec<(usize, SeatState, AgentOutcome)> {
18991        let prompts = Prompts::default();
18992        let ctx = WaveCtx {
18993            run: "r",
18994            node: "implement",
18995            prompts: &prompts,
18996            cache,
18997            round: None,
18998            carry_seats: false,
18999            pauses: None,
19000        };
19001        wave(jobs, sem, &ctx, st, 0).await
19002    }
19003
19004    #[tokio::test]
19005    async fn verification_cache_wait_preserves_the_command_budget() {
19006        crate::run::pin_test_home();
19007        let tmp = tempfile::tempdir().unwrap();
19008        let cache = tmp.path().join("target");
19009        let guard = hold(&cache);
19010        let release = tokio::spawn(async move {
19011            tokio::time::sleep(Duration::from_millis(100)).await;
19012            guard.release();
19013        });
19014        let mut st = state(10);
19015        let budget = Duration::from_millis(50);
19016        let wait = repository_wait(&st, None);
19017        let (outcomes, _) = with_cache_lease(
19018            &mut st,
19019            Some(&cache),
19020            "gate",
19021            "gate",
19022            tmp.path(),
19023            "abc1234",
19024            budget,
19025            "test gate",
19026            &wait,
19027            |_, available| async move {
19028                assert_eq!(
19029                    available, budget,
19030                    "contention must not shorten verification"
19031                );
19032                (Vec::new(), false, Vec::new())
19033            },
19034        )
19035        .await
19036        .unwrap();
19037        release.await.unwrap();
19038        assert!(outcomes.is_empty(), "{outcomes:?}");
19039        assert!(
19040            st.events
19041                .iter()
19042                .any(|event| event.message.contains("waiting for"))
19043        );
19044        assert!(st.resource_wait.is_none());
19045        // The verification body no longer owns the lease after returning.
19046        hold(&cache).release();
19047    }
19048
19049    #[tokio::test]
19050    async fn verification_cache_wait_expires_without_starting_commands() {
19051        crate::run::pin_test_home();
19052        let tmp = tempfile::tempdir().unwrap();
19053        let cache = tmp.path().join("target");
19054        let _held = hold(&cache);
19055        let mut st = state(0);
19056        let wait = repository_wait(&st, None);
19057        let (outcomes, _) = with_cache_lease(
19058            &mut st,
19059            Some(&cache),
19060            "gate",
19061            "gate",
19062            tmp.path(),
19063            "abc1234",
19064            Duration::from_secs(60),
19065            "test gate",
19066            &wait,
19067            |_, _| async { panic!("commands must not start without the cache lease") },
19068        )
19069        .await
19070        .unwrap();
19071        assert_eq!(outcomes.len(), 1);
19072        assert!(outcomes[0].resource_blocked);
19073        assert!(st.resource_wait.is_none());
19074    }
19075
19076    #[tokio::test]
19077    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
19078        crate::run::pin_test_home();
19079        let tmp = tempfile::tempdir().unwrap();
19080        let cache = tmp.path().join("target-a");
19081        let guard = hold(&cache);
19082        // Held for longer than the agent's whole 1s budget.
19083        tokio::spawn(async move {
19084            tokio::time::sleep(Duration::from_millis(1500)).await;
19085            guard.release();
19086        });
19087        let mut st = state(60);
19088        let results = run(
19089            vec![job(
19090                sh("alpha", "echo ok"),
19091                tmp.path(),
19092                Duration::from_secs(1),
19093            )],
19094            Arc::new(Semaphore::new(1)),
19095            Some(&cache),
19096            &mut st,
19097        )
19098        .await;
19099        assert!(
19100            matches!(results[0].2, AgentOutcome::Ok(_)),
19101            "the wait must not be charged to the agent"
19102        );
19103        assert!(st.resource_block.is_none());
19104        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19105        assert!(st.resource_wait.is_none() && st.active.is_empty());
19106    }
19107
19108    #[tokio::test]
19109    #[cfg(unix)]
19110    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19111        use std::os::fd::AsRawFd as _;
19112        crate::run::pin_test_home();
19113        let tmp = tempfile::tempdir().unwrap();
19114        crate::git::git(tmp.path(), &["init"]).await.unwrap();
19115        let common = tmp.path().join(".git");
19116        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19117        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19118        assert_eq!(
19119            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19120            0
19121        );
19122        let mut st = state(1);
19123        st.config.coordination = crate::config::test_coordination();
19124        let mut reader = job(
19125            sh("reader", "echo must-not-run"),
19126            tmp.path(),
19127            Duration::from_secs(30),
19128        );
19129        reader.allow_write = false;
19130        let results = run(
19131            vec![reader.clone()],
19132            Arc::new(Semaphore::new(1)),
19133            None,
19134            &mut st,
19135        )
19136        .await;
19137        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19138        assert_eq!(results[0].1.turns, 0);
19139        assert!(FailClass::of(&results[0].2).is_none());
19140        assert_eq!(
19141            st.resource_block.as_ref().unwrap().kind,
19142            "repository coordination"
19143        );
19144        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19145        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19146        drop(external);
19147        let editor = crate::repo_lock::SeatLease::for_checkout(
19148            tmp.path(),
19149            true,
19150            crate::config::test_coordination().as_ref(),
19151        )
19152        .await
19153        .unwrap();
19154        let pause = Pause::new();
19155        let interrupt = Pause::new();
19156        let prompts = Prompts::default();
19157        let ctx = WaveCtx {
19158            run: "r",
19159            node: "review",
19160            prompts: &prompts,
19161            cache: None,
19162            round: None,
19163            carry_seats: false,
19164            pauses: Some((&pause, &interrupt)),
19165        };
19166        let (results, ()) = tokio::join!(
19167            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19168            async {
19169                tokio::time::sleep(Duration::from_millis(50)).await;
19170                pause.park();
19171            }
19172        );
19173        assert!(
19174            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19175        );
19176        assert_eq!(results[0].1.turns, 0);
19177        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19178        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19179        drop(editor);
19180    }
19181
19182    #[tokio::test]
19183    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19184        crate::run::pin_test_home();
19185        let tmp = tempfile::tempdir().unwrap();
19186        let cache = tmp.path().join("target-b");
19187        let _held = hold(&cache);
19188        let mut st = state(1);
19189        let results = run(
19190            vec![job(
19191                sh("alpha", "echo ok"),
19192                tmp.path(),
19193                Duration::from_secs(1),
19194            )],
19195            Arc::new(Semaphore::new(1)),
19196            Some(&cache),
19197            &mut st,
19198        )
19199        .await;
19200        let (_, seat, out) = &results[0];
19201        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19202        assert!(wave_blocked(&results));
19203        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19204        assert_eq!(seat.turns, 0, "nothing was asked");
19205        assert!(
19206            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19207            "no agent was spawned"
19208        );
19209        assert!(FailClass::of(out).is_none(), "not a failure class");
19210        let block = st.resource_block.as_ref().expect("recorded for the resume");
19211        assert_eq!(block.kind, "shared build cache");
19212        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19213        assert!(st.active.is_empty());
19214    }
19215
19216    #[tokio::test]
19217    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19218        crate::run::pin_test_home();
19219        let tmp = tempfile::tempdir().unwrap();
19220        let mut st = state(1);
19221        let results = run(
19222            vec![job(
19223                sh("alpha", "echo ok"),
19224                tmp.path(),
19225                Duration::from_secs(1),
19226            )],
19227            Arc::new(Semaphore::new(0)),
19228            None,
19229            &mut st,
19230        )
19231        .await;
19232        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19233        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19234        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19235    }
19236
19237    #[tokio::test]
19238    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19239        crate::run::pin_test_home();
19240        let tmp = tempfile::tempdir().unwrap();
19241        let mut st = state(60);
19242        let results = run(
19243            vec![job(
19244                sh("alpha", "sleep 30"),
19245                tmp.path(),
19246                Duration::from_secs(1),
19247            )],
19248            Arc::new(Semaphore::new(1)),
19249            None,
19250            &mut st,
19251        )
19252        .await;
19253        let out = &results[0].2;
19254        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19255        let class = FailClass::of(out).expect("a failure");
19256        assert_eq!(class, FailClass::Timeout);
19257        assert!(should_hand_over(None, &class), "existing handover policy");
19258        assert!(st.resource_block.is_none());
19259    }
19260}