Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// How much verification output is kept and fed back to the fixer.
64const OUTPUT_TAIL: usize = 8_000;
65
66/// Bytes of a failing command's output kept in an event, so the reason a run
67/// stopped is readable from the report without opening `run.json`.
68const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
71/// command's pid before releasing the build cache's lease.
72const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
75/// command's pid to actually exit before giving up and releasing anyway.
76///
77/// A timeout means the process was asked to die (`kill_on_drop`,
78/// `start_kill`), not that it already has — on Windows in particular that can
79/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
80/// the instant the command returns would let the very next acquirer (this
81/// run's own next round, another run's verification, the janitor's prune)
82/// start touching the same directory while it might still be writing to it,
83/// so this polls the actual pid — real confirmation, not a fixed guess —
84/// until it is gone or this ceiling is reached. It is still not full
85/// process-tree reaping: a grandchild the timed-out process spawned and that
86/// outlives it independently is invisible to a pid check, and continuing to
87/// observe and collect *that* stays a different piece of work with its own
88/// owner. Set generously because the common case returns early the moment
89/// the pid is confirmed gone, not because every timeout pays this in full.
90const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92/// Consecutive review rounds with no tree progress (see
93/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
94/// instead of spending the rest of the round budget.
95///
96/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
97/// legitimately finds nothing left to change (its previous round's fix already
98/// covered it, and this round's reviewers re-raised only nits) looks the same
99/// as one that is spinning, for exactly one round. Two in a row is where the
100/// two stop being distinguishable, and a review round on this workload has
101/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
102/// third attempt at a tree that has not moved twice running is pure cost.
103/// This does not touch `review_rounds` itself, which stays the operator's
104/// call.
105pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
108/// a base that moved before giving up and leaving the run `Blocked` for a
109/// person.
110///
111/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
112/// that keeps moving faster than a run can catch it is not something more
113/// rebasing fixes, it is a person's call. Not the same *number as*
114/// `land_rounds` - this budget is spent before a pull request exists, land's
115/// after - but bounded for the identical reason, so it uses the same
116/// default. Counted across both call sites in [`Runner::finish_after_tally`]
117/// (once before review, once before the gate), because either one finding
118/// the base still moving is the same signal.
119const BASE_SYNC_ROUNDS: usize = 4;
120
121/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
122/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
123/// reply held no [`FixReport`].
124///
125/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
126/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
127/// "I'll pause here until the `cargo make check` background run reports
128/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
129/// No `FixReport` was ever collected from that seat, and the run moved on to
130/// the next review round regardless.
131///
132/// Bounded independently of `review_rounds` and `graph.retries`: this
133/// recovers one seat's missing report mid-round, not a new round of review or
134/// an ordinary parse retry, and must not itself become the unbounded wait the
135/// rest of this module exists to avoid.
136const MAX_FIX_CONTINUATIONS: usize = 2;
137
138/// One queued agent invocation.
139///
140/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
141/// CLI hung up on its own stream is asked again from the same job rather than
142/// rebuilt from scratch. See [`Runner::resume_undelivered`].
143#[derive(Clone)]
144struct SeatJob {
145    spec: AgentSpec,
146    seat: SeatState,
147    cwd: PathBuf,
148    prompt: String,
149    timeout: Duration,
150    allow_write: bool,
151    sessions: bool,
152    artifacts: PathBuf,
153    stem: String,
154    /// The prompt for a seat that has been handed to another roster agent
155    /// (a fresh session): everything the original seat would have
156    /// remembered. `None` when `prompt` already carries it, as the first
157    /// ranking and the implement prompt do. Never a resume-style prompt.
158    handover: Option<String>,
159}
160
161/// How the graph reads one agent invocation.
162///
163/// Quota is split out from an ordinary failure on purpose: a rate-limited call
164/// is known to fail again if retried now, so the retry loop must not spend an
165/// attempt on it. `Dropped` is split out for the opposite reason: unlike
166/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
167/// error JSON, never the agent's answer — a caller that matched only
168/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
169/// left to read that JSON as if it were usable output. `resume_undelivered`
170/// is the only caller that acts on it; everywhere else it is reported like an
171/// ordinary failure.
172enum AgentOutcome {
173    /// A usable output.
174    Ok(AgentOutput),
175    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
176    Quota(AgentOutput),
177    /// The CLI hung up on its own stream after billed work. See
178    /// [`agent::AgentOutput::work_undelivered`].
179    Dropped(AgentOutput),
180    /// Any other failure: a timeout, a bad exit code, an empty reply.
181    Failed(String),
182    /// A resource prevented admission or interrupted execution. Not an agent
183    /// failure: never advance a fallback chain or spend an attempt on it.
184    /// Post-spawn interruptions carry a typed error into wave, which records
185    /// implementation preservation before returning this outcome. This outcome
186    /// alone is not proof that no source changes were made.
187    /// [`RunState::resource_block`] says why.
188    ResourceBlocked(String),
189}
190
191/// A request to park the run at its next node boundary.
192///
193/// Cloning is how the request travels: the loop keeps one handle and hands a
194/// clone to each [`Runner`], and every clone points at the same flag. There
195/// is no channel because there is nothing to send - the only message is
196/// "park", it is idempotent, and a flag cannot be missed by a receiver that
197/// was not listening yet.
198///
199/// The boundary is what makes this cheap. Every node writes the run's state
200/// before the next one starts, and every node skips what is already recorded:
201/// `prep` returns early once candidates exist, `implement` asks only the seats
202/// with nothing on disk, `judge` returns early once judgements exist. So a
203/// parked run resumes into exactly the node it stopped before, and no agent
204/// work is thrown away. Killing the process mid-node, by contrast, loses
205/// whatever the seats in flight had not yet written - which for an implement
206/// wave is an hour of paid work.
207///
208/// A [`Runner`] watches two independent handles of this type - see
209/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
210/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
211/// clone covering the whole daemon's lifetime and is never asked to un-park,
212/// which is correct exactly because nothing is dispatched after it fires.
213/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
214/// that parks for an interrupted task must go on to run other tasks
215/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
216/// reusing the daemon-wide one.
217#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221    /// A pause nobody has asked for yet.
222    #[must_use]
223    pub fn new() -> Self {
224        Self::default()
225    }
226
227    /// Ask the run to park at its next node boundary. Idempotent.
228    pub fn park(&self) {
229        self.0.store(true, Ordering::SeqCst);
230    }
231
232    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
233    /// fold into the run's own `park` event - so an operator reading the run
234    /// later knows this was a deliberate interrupt rather than a shutdown or
235    /// a binary swap. The first reason recorded wins; a park already in
236    /// flight is not relabelled by a second, unrelated request.
237    pub fn park_because(&self, reason: impl Into<String>) {
238        self.park_for(ParkCause::Interrupt {
239            reason: reason.into(),
240        });
241    }
242
243    /// [`Pause::park`], recording the structured cause. The first cause
244    /// recorded wins, like [`Pause::park_because`].
245    pub fn park_for(&self, cause: ParkCause) {
246        let mut reason_guard = self
247            .1
248            .lock()
249            .unwrap_or_else(std::sync::PoisonError::into_inner);
250        if reason_guard.is_none() {
251            *reason_guard = Some(cause);
252        }
253        drop(reason_guard);
254        self.park();
255    }
256
257    /// Has a park been asked for?
258    #[must_use]
259    pub fn parked(&self) -> bool {
260        self.0.load(Ordering::SeqCst)
261    }
262
263    /// Why the park was asked for, when the caller used [`Pause::park_because`].
264    #[must_use]
265    pub fn reason(&self) -> Option<String> {
266        self.cause().map(|c| c.label())
267    }
268
269    /// The structured cause, when one was recorded.
270    #[must_use]
271    pub fn cause(&self) -> Option<ParkCause> {
272        self.1
273            .lock()
274            .unwrap_or_else(std::sync::PoisonError::into_inner)
275            .clone()
276    }
277}
278
279/// Drives one run.
280pub struct Runner {
281    /// Run state; public so the CLI can report on it.
282    pub state: RunState,
283    roles: ResolvedRoles,
284    sem: Arc<Semaphore>,
285    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
286    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
287    /// this is never the same handle as `interrupt`.
288    pause: Pause,
289    /// Set when `magi serve`'s interrupt scheduler wants this specific run
290    /// parked at its next node boundary, to let a task marked
291    /// [`crate::queue::Task::interrupt`] run alone before this one carries
292    /// on. Unlike `pause`, a fresh, unshared handle per run - see
293    /// [`Runner::watch_interrupt`].
294    interrupt: Pause,
295}
296
297/// Where the branch's own commits start: its merge base with the base branch
298/// as the remote has it now, else with the recorded `base_commit`. A branch
299/// rebased onto a base that moved past `base_commit` would otherwise count
300/// the base's commits as its own under `base_commit..branch`.
301async fn review_base(
302    repo: &Path,
303    remote: &str,
304    base_branch: &str,
305    base_commit: &str,
306    branch: &str,
307) -> String {
308    review_base_checked(repo, remote, base_branch, base_commit, branch)
309        .await
310        .0
311}
312
313/// [`review_base`] plus whether the base was read from a freshly fetched
314/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
315/// never older than `base_commit`'s view of the base), but the answer is then
316/// not trusted to rewrite a pull request's title.
317async fn review_base_checked(
318    repo: &Path,
319    remote: &str,
320    base_branch: &str,
321    base_commit: &str,
322    branch: &str,
323) -> (String, bool) {
324    let tracking = format!("{remote}/{base_branch}");
325    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326    if git::rev_exists(repo, &tracking).await
327        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328        && !mb.is_empty()
329    {
330        return (mb, fresh);
331    }
332    let mb = git::merge_base(repo, base_commit, branch)
333        .await
334        .ok()
335        .filter(|mb| !mb.is_empty())
336        .unwrap_or_else(|| base_commit.to_owned());
337    (mb, false)
338}
339
340/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
341/// when git cannot say or finds nothing: a stale list is better than a wrong
342/// or empty one.
343pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344    if !is_review_run(state) {
345        return;
346    }
347    let base = review_base(
348        &state.repo,
349        &state.config.merge.remote,
350        &state.base_branch,
351        &state.base_commit,
352        branch,
353    )
354    .await;
355    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356        && !subjects.is_empty()
357        && state.reviewed_commits.as_ref() != Some(&subjects)
358    {
359        state.reviewed_commits = Some(subjects);
360        state.save().ok();
361    }
362}
363
364/// Subjects of the base's commits between the recorded start and the branch's
365/// merge base: what a stale `base_commit..branch` would have mistaken for the
366/// branch's own work.
367async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368    let (base, trusted) = review_base_checked(
369        &state.repo,
370        &state.config.merge.remote,
371        &state.base_branch,
372        &state.base_commit,
373        branch,
374    )
375    .await;
376    if !trusted {
377        return None;
378    }
379    git::subjects(&state.repo, &state.base_commit, &base)
380        .await
381        .ok()
382}
383
384/// May an adopted pull request's title be replaced with `computed`? Only when
385/// it is empty, magi's own shape, or a base commit's subject that leaked in;
386/// a title a person wrote stays. Never when `computed` is itself a leak.
387pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389    if is_leak(computed) {
390        return false;
391    }
392    let cur = current.trim();
393    cur.is_empty()
394        || cur.starts_with(REVIEW_PROMPT_OPENING)
395        || cur.starts_with("chore: land candidate")
396        || cur.starts_with("magi: candidate")
397        || is_leak(cur)
398}
399
400/// The commit a run branches from: the base branch as the remote has it.
401///
402/// Two failures this replaces. A run used to branch off `HEAD` and so refused
403/// to start on a dirty tree, which made `magi serve` decline every task for as
404/// long as the operator had work in progress - most of the time. Branching off
405/// the *local* base branch fixed that and introduced a worse one: `land` merges
406/// the winner on GitHub, nothing updates the local ref, and the next run
407/// branches off a base missing everything the previous runs landed. Two tasks
408/// in a row from a phone would have had the second silently re-implementing
409/// against stale code and opening a pull request that reverted the first.
410///
411/// Only refs move here - no checkout, no local branch, no merge - so it is safe
412/// with uncommitted work in the tree. A machine with no network still starts:
413/// the fetch may fail and the local tip is used with a warning, because
414/// refusing to run offline is a worse failure than running against a base the
415/// operator can see for themselves.
416///
417/// One function, called by both entry points. Two answers to "where does a run
418/// branch from" is the kind of drift nobody notices until a diff is wrong.
419/// Bring the local `branch` in line with `<remote>/<branch>` before a review
420/// checks it out.
421///
422/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
423/// anything other than plain `git push` from this checkout (a jj colocated
424/// workspace, another clone) moves only the remote-tracking ref - so the local
425/// one can be a stale placeholder. It moves only when local is behind the remote or is an
426/// empty placeholder that diverged from it; unpushed local work is kept, and a real
427/// divergence is refused rather than guessed at.
428async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429    let tracking = format!("{remote}/{branch}");
430    let fetched = git::fetch(repo, remote, branch).await;
431    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432    let local_exists = git::branch_exists(repo, branch).await?;
433    if !fresh {
434        if !local_exists {
435            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436        }
437        tracing::warn!(
438            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439        );
440        return Ok(());
441    }
442    let remote_sha = git::rev_parse(repo, &tracking).await?;
443    if !local_exists {
444        git::git(repo, &["branch", branch, &tracking]).await?;
445        return Ok(());
446    }
447    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449        return Ok(());
450    }
451    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452        // Diverged. `reconcile` settles it only when it can prove nothing is
453        // lost: a local tip that is the remote's change rebased is pushed over
454        // it (lease pinned to the tip read here), a tip whose every extra
455        // commit is empty is a placeholder the remote's work replaced, and
456        // anything else is two different changes - a question for a person.
457        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458            .await?
459        {
460            crate::reconcile::Reconciliation::Pushed => {
461                tracing::warn!(
462                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463                    short(&local_sha),
464                    short(&remote_sha)
465                );
466                return Ok(());
467            }
468            crate::reconcile::Reconciliation::Placeholder => {}
469            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470        }
471    }
472    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473    if !out.ok() {
474        bail!(
475            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476            short(&local_sha),
477            short(&remote_sha),
478            out.stderr
479        );
480    }
481    tracing::warn!(
482        "local `{branch}` was stale: fast-forwarded {} -> {}",
483        short(&local_sha),
484        short(&remote_sha)
485    );
486    Ok(())
487}
488
489async fn resolve_base(
490    repo: &Path,
491    base_branch: &str,
492    remote: &str,
493    coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495    let _lease = crate::repo_lock::SeatLease::operation_with(
496        repo,
497        coordination,
498        &crate::repo_lock::WaitPolicy::current(),
499    )
500    .await?;
501    let tracking = format!("{remote}/{base_branch}");
502    let fetched = git::fetch(repo, remote, base_branch).await;
503    if let Ok(out) = &fetched
504        && out.ok()
505        && git::rev_exists(repo, &tracking).await
506    {
507        return Ok(crate::run::FetchedBase {
508            remote: remote.to_owned(),
509            branch: base_branch.to_owned(),
510            commit: git::rev_parse(repo, &tracking).await?,
511            fetched_at: jiff::Timestamp::now(),
512        });
513    }
514    let why = match &fetched {
515        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516        Ok(_) => format!("{remote} has no {base_branch}"),
517        Err(e) => e.to_string(),
518    };
519    // No fallback to the local branch: it may be behind, and branching off an
520    // old commit is the stale-checkout bug this check exists to prevent.
521    bail!(
522        "cannot read {tracking} ({why}); refusing to branch off the local \
523         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524         base / remote in magi.toml. This is launcher preparation: check the \
525         launcher's Git administration access, network and repository coordination \
526         before retrying; an unattended seat cannot approve those permissions"
527    )
528}
529
530/// Outcome of the shared fixer chain, before caller-owned stop policy.
531enum FixStep {
532    Resource,
533    Recorded { report_lost: bool },
534}
535
536/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
537/// on an early return or a panic.
538///
539/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
540/// manual `magi fix` invocations against the same run are otherwise
541/// invisible to each other and would race to remove and recreate the same
542/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
543/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
544/// which is only ever reclaimed later, out of band, by
545/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
546/// `magi fix` invocation is not necessarily running under either of those, so
547/// nothing would ever sweep a lock a killed or crashed process left behind.
548/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
549/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
550/// lease use: an unreadable or unparsable pid, or a liveness query the
551/// platform cannot answer, reads as alive and the lock is left in place.
552struct FixClaim {
553    path: PathBuf,
554}
555
556impl FixClaim {
557    fn acquire(dir: &Path) -> Result<Self> {
558        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
559        let path = dir.join("fix.lock");
560        let retry = crate::lock_file::Retry::new(None);
561        match Self::create(&path, retry) {
562            Ok(claim) => Ok(claim),
563            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
564                if Self::reclaim_if_dead(&path) {
565                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
566                } else {
567                    bail!(
568                        "another repair or `magi fix` is already running for this run ({} exists)",
569                        path.display()
570                    )
571                }
572            }
573            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
574        }
575    }
576
577    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
578        let mut f = retry.create(path)?;
579        use std::io::Write as _;
580        // Read back by `reclaim_if_dead` on a later, stuck invocation.
581        writeln!(f, "{}", std::process::id())?;
582        Ok(Self {
583            path: path.to_owned(),
584        })
585    }
586
587    /// True if the lock named a process confirmed dead, in which case it was
588    /// also removed. Never true on an unreadable file, an unparsable pid, or
589    /// a liveness query the platform cannot answer — see this type's own doc.
590    fn reclaim_if_dead(path: &Path) -> bool {
591        let dead = std::fs::read_to_string(path)
592            .ok()
593            .and_then(|body| body.trim().parse::<u32>().ok())
594            .is_some_and(|pid| !crate::proc::pid_alive(pid));
595        dead && std::fs::remove_file(path).is_ok()
596    }
597}
598
599impl Drop for FixClaim {
600    fn drop(&mut self) {
601        let _ = std::fs::remove_file(&self.path);
602    }
603}
604
605/// Holds the mutable runner as well as its durable lease: cancellation must
606/// clear both copies of ownership, including state a caller may save later.
607struct DriverExecution<'a> {
608    runner: &'a mut Runner,
609    driver: crate::driver::DriverGuard,
610}
611
612impl Drop for DriverExecution<'_> {
613    fn drop(&mut self) {
614        self.driver.release_local(&mut self.runner.state);
615    }
616}
617
618impl Runner {
619    /// Start a fresh run against `repo`.
620    pub async fn start(
621        repo: &Path,
622        instruction: String,
623        config: Config,
624        origin: Origin,
625    ) -> Result<Self> {
626        Self::start_naming(repo, instruction, "", config, origin).await
627    }
628
629    /// [`Runner::start`] for a queued task. Titles and instructions never
630    /// select implementation sources. The title argument is retained for callers.
631    pub async fn start_naming(
632        repo: &Path,
633        instruction: String,
634        _also_scan: &str,
635        config: Config,
636        origin: Origin,
637    ) -> Result<Self> {
638        let repo = git::toplevel(repo).await?;
639        let missing = agent::missing_programs(&config.agents);
640        if !missing.is_empty() {
641            bail!(
642                "these agent programs are not on PATH: {}. Fix the roster in \
643                 magi.toml or install them.",
644                missing.join(", ")
645            );
646        }
647        let base_branch =
648            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
649                .await?;
650        let fetched_base = resolve_base(
651            &repo,
652            &base_branch,
653            &config.merge.remote,
654            config.coordination.as_ref(),
655        )
656        .await?;
657        let base_commit = fetched_base.commit.clone();
658        let roles = config.resolve_roles()?;
659        let max_parallel = config.graph.max_parallel.max(1);
660        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
661        // Recorded before the first save, so a crash right after minting
662        // cannot leave a run with no origin. Legibility only: nothing reads it
663        // to decide anything.
664        state.fetched_base = Some(fetched_base);
665        state.origin = Some(origin);
666        state.reserve()?;
667        state.event("start", format!("run {} created", state.id));
668        state.save()?;
669        Ok(Self {
670            state,
671            roles,
672            sem: Arc::new(Semaphore::new(max_parallel)),
673            pause: Pause::new(),
674            interrupt: Pause::new(),
675        })
676    }
677
678    /// Open a review-only run against work that already exists on `branch`.
679    ///
680    /// The expensive half of the graph is the implement wave — measured at
681    /// 111 and 134 internal tool-loop turns on this repository, against a
682    /// handful for a judge or a reviewer. The cheap half is worth running on
683    /// hand-written work too, and there was no way to reach it.
684    ///
685    /// No new state and no schema change are needed: a run with **one** viable
686    /// candidate and a tally already decided degrades `execute` to exactly
687    /// review → gate → merge, because `judge` skips a single-candidate field,
688    /// `deliberate` has fewer than two first choices to reconcile, `vote`
689    /// returns early, `tally` is already present and `fold_losers` has no
690    /// losers. Resuming such a run therefore does the right thing as well.
691    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
692        Self::review_taking_over(repo, branch, config, None, origin).await
693    }
694
695    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
696    /// at the same task still has `branch` checked out, its worktree is
697    /// released first if that is safe (see [`crate::handover`]), and the
698    /// review refuses with the reason if it is not. `None` is a hand-run
699    /// review: it has no earlier attempts, so only a worktree of a dead run
700    /// magi recorded itself can be released.
701    pub async fn review_taking_over(
702        repo: &Path,
703        branch: &str,
704        config: Config,
705        takeover: Option<crate::handover::Takeover>,
706        origin: Origin,
707    ) -> Result<Self> {
708        let repo = git::toplevel(repo).await?;
709        let missing = agent::missing_programs(&config.agents);
710        if !missing.is_empty() {
711            bail!(
712                "these agent programs are not on PATH: {}. Fix the roster in \
713                 magi.toml or install them.",
714                missing.join(", ")
715            );
716        }
717        let base_branch =
718            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
719                .await?;
720        if base_branch == branch {
721            bail!("`{branch}` is the base branch; there is nothing to review against");
722        }
723        let fetched_base = resolve_base(
724            &repo,
725            &base_branch,
726            &config.merge.remote,
727            config.coordination.as_ref(),
728        )
729        .await?;
730        let base_commit = fetched_base.commit.clone();
731
732        let roles = config.resolve_roles()?;
733        let max_parallel = config.graph.max_parallel.max(1);
734        let mut state = RunState::new(
735            repo.clone(),
736            base_branch,
737            base_commit.clone(),
738            String::new(),
739            config,
740        );
741        state.fetched_base = Some(fetched_base);
742        state.origin = Some(origin);
743        state.reserve()?;
744
745        // Released before anything else touches the branch: a stale local
746        // branch is moved with `git branch -f`, which git refuses while an
747        // earlier attempt's worktree still has it checked out. Everything
748        // after this point that can fail puts the old run back.
749        // A hand-run review has no task, hence no earlier attempts, but a
750        // worktree of a dead run magi made may still be released.
751        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
752            earlier: Vec::new(),
753            home: crate::run::home(),
754            choice: None,
755            task_instruction: None,
756        });
757        let wait = repository_wait(&state, None);
758        let coordination_config = state.config.coordination.clone();
759        let coordination =
760            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
761                .await?;
762        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
763        if let Some(released) = &released {
764            state.event(
765                "release",
766                format!(
767                    "took `{branch}` over from run {}: its worktree was released: {}",
768                    crate::run::short_of(&released.old_id),
769                    released.audit
770                ),
771            );
772        }
773        // The owner's answer to an earlier divergence question is applied
774        // here: after the release (git will not move a checked-out branch)
775        // and before the sync that would otherwise ask again.
776        if let Some(choice) = takeover.choice.as_ref()
777            && let Err(e) =
778                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
779                    .await
780        {
781            if let Some(released) = &released {
782                released.restore(&repo, branch).await;
783            }
784            return Err(e.context("applying the owner's answer about the diverged branch"));
785        }
786        // An earlier attempt of this task that published the task head, and
787        // whose work is the branch under review, hands that head over: the
788        // review run updates the same pull request. Without one (a legacy
789        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
790        // own head and its own open pull request is adopted as before.
791        for id in takeover.earlier.iter().rev() {
792            if let Ok(old) = RunState::load_under(id, &takeover.home)
793                && let Some(head) = old.pr_head.clone()
794                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
795            {
796                state.pr_head = Some(head);
797                break;
798            }
799        }
800        drop(coordination);
801        let opened = Self::open_review(
802            &repo,
803            branch,
804            state,
805            roles,
806            max_parallel,
807            base_commit,
808            &takeover,
809        )
810        .await;
811        if opened.is_err()
812            && let Some(released) = &released
813        {
814            let _coordination = crate::repo_lock::SeatLease::operation_with(
815                &repo,
816                coordination_config.as_ref(),
817                &wait,
818            )
819            .await?;
820            released.restore(&repo, branch).await;
821        }
822        opened
823    }
824
825    /// Recover exactly the operator-selected public implementation, without touching old sources.
826    pub async fn recover_pr(
827        repo: &Path,
828        task: &str,
829        selection: crate::pr_recovery::Selection,
830        mut config: Config,
831    ) -> Result<Self> {
832        selection.configure(&mut config)?;
833        selection.check_task(task, &selection.instruction)?;
834        let base_branch =
835            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
836                .await?;
837        let fetched_base = resolve_base(
838            repo,
839            &base_branch,
840            &config.merge.remote,
841            config.coordination.as_ref(),
842        )
843        .await?;
844        let base_commit = fetched_base.commit.clone();
845        selection
846            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
847            .await?;
848        Box::pin(selection.verify_source(repo)).await?;
849        if let Some(proof) = &selection.source_proof {
850            anyhow::ensure!(
851                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
852                "source rebase base is outside the fetched landing base"
853            );
854        }
855        let roles = config.resolve_roles()?;
856        let max_parallel = config.graph.max_parallel.max(1);
857        let mut state = RunState::new(
858            repo.to_path_buf(),
859            base_branch,
860            base_commit.clone(),
861            selection.instruction.clone(),
862            config,
863        );
864        state.fetched_base = Some(fetched_base);
865        state.origin = Some(crate::run::Origin::queue(task));
866        state.pr_head = Some(selection.branch.clone());
867        state.pr_recovery = Some(selection.clone());
868        state.reserve()?;
869        if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
870            // Never leave a reserved record behind as a phantom owner of the head.
871            state.discard_reserved();
872            return Err(e);
873        }
874        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
875    }
876
877    async fn open_selected(
878        repo: &Path,
879        state: RunState,
880        roles: ResolvedRoles,
881        max_parallel: usize,
882        base_commit: String,
883        selection: crate::pr_recovery::Selection,
884    ) -> Result<Self> {
885        let mut state = state;
886        let branch = state.branch_for('A');
887        let wait = repository_wait(&state, None);
888        let prepared = async {
889            let coordination = coordinate_operation(&mut state, repo, &wait).await?;
890            git::git(repo, &["branch", &branch, selection.source()]).await?;
891            drop(coordination);
892            anyhow::Ok(())
893        }
894        .await;
895        if let Err(e) = prepared {
896            // Nothing was exposed yet: do not leave a reserved record as a
897            // phantom owner of the public head.
898            state.discard_reserved();
899            return Err(e);
900        }
901        state.event(
902            "recovery",
903            format!(
904                "selected implementation {} for {} (public lease {})",
905                selection.source(),
906                selection.url,
907                selection.head
908            ),
909        );
910        let takeover = crate::handover::Takeover {
911            earlier: Vec::new(),
912            home: crate::run::home(),
913            choice: None,
914            task_instruction: Some(selection.instruction.clone()),
915        };
916        if let Err(e) = state.save() {
917            state.discard_reserved();
918            return Err(e);
919        }
920        let id = state.id.clone();
921        let opened = Self::open_review(
922            repo,
923            &branch,
924            state,
925            roles,
926            max_parallel,
927            base_commit,
928            &takeover,
929        )
930        .await;
931        match opened {
932            Ok(runner) => Ok(runner),
933            Err(e) => {
934                if let Ok(mut partial) = RunState::load(&id) {
935                    partial.status = RunStatus::Blocked;
936                    partial.driver_exited = true;
937                    partial.event(
938                        "recovery",
939                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
940                    );
941                    partial.save()?;
942                }
943                Err(crate::pr_recovery::StartupFailed {
944                    run: id,
945                    cause: format!("{e:#}"),
946                }
947                .into())
948            }
949        }
950    }
951
952    /// What the newest earlier attempt that reviewed `branch` leaves for this
953    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
954    /// no earlier attempts. Reads the earlier record only: ownership of its
955    /// worktree and branch is [`crate::handover`]'s business.
956    async fn inherit(
957        repo: &Path,
958        branch: &str,
959        base_commit: &str,
960        takeover: &crate::handover::Takeover,
961    ) -> Option<crate::recovery::RecoveryRecord> {
962        use crate::recovery::{Relation, assess};
963        let newest = takeover.earlier.last()?;
964        let mut chosen = None;
965        for id in takeover.earlier.iter().rev() {
966            if let Ok(old) = RunState::load_under(id, &takeover.home)
967                && !old.reviews.is_empty()
968                && (old.winner().is_some_and(|w| w.branch == branch)
969                    || old.pr_head.as_deref() == Some(branch))
970            {
971                chosen = Some(old);
972                break;
973            }
974        }
975        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
976            .await
977            .unwrap_or_default();
978        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
979            Some(last) if last.head == tip => Relation::Same,
980            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
981            _ => Relation::Diverged,
982        };
983        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
984        Some(assess(from, chosen.as_ref(), base_commit, relation))
985    }
986
987    /// The half of [`Runner::review_taking_over`] that can fail after an
988    /// earlier attempt's worktree was released.
989    async fn open_review(
990        repo: &Path,
991        branch: &str,
992        mut state: RunState,
993        roles: ResolvedRoles,
994        max_parallel: usize,
995        base_commit: String,
996        takeover: &crate::handover::Takeover,
997    ) -> Result<Self> {
998        let wait = repository_wait(&state, None);
999        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1000        if state.pr_recovery.is_none() {
1001            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1002        }
1003        // The commit subjects are the closest thing to a task statement that
1004        // existing work carries, and the reviewers are told as much.
1005        let start = review_base(
1006            repo,
1007            &state.config.merge.remote,
1008            &state.base_branch,
1009            &base_commit,
1010            branch,
1011        )
1012        .await;
1013        let log = git::log_oneline(repo, &start, branch)
1014            .await
1015            .unwrap_or_default();
1016        let log_text = if log.trim().is_empty() {
1017            "(no commit messages)"
1018        } else {
1019            log.trim()
1020        };
1021        let mut instruction = format!(
1022            "Review the work already on branch `{branch}`. There is no task \
1023             statement: what the change claims to do is whatever its commits \
1024             say.\n\n{log_text}"
1025        );
1026        // A retry of a task whose earlier attempt left reviews behind carries
1027        // that attempt's evidence, and the task's own statement with it: the
1028        // completion criteria are the task's, not whatever the commits claim.
1029        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1030            if let Some(task) = takeover
1031                .task_instruction
1032                .as_deref()
1033                .filter(|t| !t.trim().is_empty())
1034                && rec.reviewed_head.is_some()
1035            {
1036                instruction = format!(
1037                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1038                    task.trim()
1039                );
1040                rec.criteria_inherited = true;
1041            }
1042            state.event("recovery", rec.summary());
1043            state.recovery = Some(rec);
1044        }
1045        if let Some(selected) = &state.pr_recovery {
1046            instruction = selected.instruction.clone();
1047        }
1048        state.task_instruction = takeover.task_instruction.clone();
1049        state.instruction = instruction;
1050        state.reviewed_commits = Some(
1051            git::subjects(repo, &start, branch)
1052                .await
1053                .unwrap_or_default(),
1054        );
1055
1056        // A takeover must not forget a hold that a previous review observed,
1057        // even if repairs have since removed it from the branch.
1058        for id in &takeover.earlier {
1059            if let Ok(old) = RunState::load_under(id, &takeover.home)
1060                && old.repo == state.repo
1061                && (old.pr_head.as_deref() == Some(branch)
1062                    || old.winner().is_some_and(|w| w.branch == branch))
1063            {
1064                for hold in old.branch_owner_holds {
1065                    if !state.branch_owner_holds.contains(&hold) {
1066                        state.event(
1067                            "approval-policy",
1068                            format!(
1069                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1070                                hold.control.key(),
1071                                hold.file,
1072                                hold.head
1073                            ),
1074                        );
1075                        state.branch_owner_holds.push(hold);
1076                    }
1077                }
1078            }
1079        }
1080        let reviewed_head = git::rev_parse(repo, branch).await?;
1081        state.observe_branch_owner_holds(&reviewed_head).await;
1082
1083        // An attached worktree, so the fixer's commits land on the branch under
1084        // review rather than on a detached head nobody will look at again.
1085        let worktree = state.worktree_root().join("under-review");
1086        if let Some(parent) = worktree.parent() {
1087            tokio::fs::create_dir_all(parent).await.ok();
1088        }
1089        let path = worktree.to_string_lossy().to_string();
1090        git::git(repo, &["worktree", "add", &path, branch])
1091            .await
1092            .with_context(|| {
1093                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1094            })?;
1095
1096        let setup = match ensure_setup_config(&mut state, repo).await {
1097            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1098            Err(e) => Err(e),
1099        };
1100        if let Err(e) = setup {
1101            git::worktree_remove_clean(repo, &worktree).await.ok();
1102            return Err(e);
1103        }
1104        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1105            .await
1106            .unwrap_or(0);
1107        if commits == 0 {
1108            git::worktree_remove_clean(repo, &worktree).await.ok();
1109            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1110        }
1111        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1112            .await
1113            .map(|f| f.len())
1114            .unwrap_or(0);
1115        if files == 0
1116            && let (Ok(head_tree), Ok(base_tree)) = (
1117                git::tree_of(&worktree, "HEAD").await,
1118                git::tree_of(&worktree, &base_commit).await,
1119            )
1120            && head_tree == base_tree
1121        {
1122            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1123            git::worktree_remove_clean(repo, &worktree).await.ok();
1124            bail!(
1125                "`{branch}` at {} has a tree identical to base {}; this usually means \
1126                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1127                 against `{}/{branch}`) rather than an empty change",
1128                short(&head),
1129                short(&base_commit),
1130                state.config.merge.remote
1131            );
1132        }
1133        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1134            .await
1135            .unwrap_or_default();
1136
1137        state.candidates.push(Candidate {
1138            index: 0,
1139            label: 'A',
1140            // Not an agent id on purpose: nothing in the roster wrote this, and
1141            // the stats tables must not credit anyone with a win for it.
1142            agent: EXISTING_BRANCH.to_owned(),
1143            branch: branch.to_owned(),
1144            worktree,
1145            summary: String::new(),
1146            stat,
1147            files,
1148            commits,
1149            empty: false,
1150            failed: None,
1151            verified_noop: None,
1152            duration_ms: 0,
1153            folded: false,
1154        });
1155        state.tally = Some(Tally {
1156            first_choice: BTreeMap::from([('A', 0)]),
1157            borda: BTreeMap::new(),
1158            winner: 'A',
1159            rankings: 0,
1160            unanimous_initial: false,
1161            deliberated: false,
1162            changed_votes: 0,
1163            unanimous_final: false,
1164            tie_break: None,
1165            // No panel sat, so no quorum applies. Zero judges is the correct
1166            // number for work that never competed, and must not be reported as
1167            // a collapsed panel.
1168            judges: 0,
1169            present: 0,
1170            quorum: 0,
1171            met_quorum: true,
1172            uncontested: Some("review-only run: nothing competed".to_owned()),
1173        });
1174        state.status = RunStatus::Reviewing;
1175        state.prep_complete = Some(true);
1176        state.event(
1177            "start",
1178            format!(
1179                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1180                state.id
1181            ),
1182        );
1183        state.save()?;
1184        Ok(Self {
1185            state,
1186            roles,
1187            sem: Arc::new(Semaphore::new(max_parallel)),
1188            pause: Pause::new(),
1189            interrupt: Pause::new(),
1190        })
1191    }
1192
1193    /// Reopen an existing run.
1194    pub fn resume(id: &str) -> Result<Self> {
1195        let state = RunState::load(id)?;
1196        crate::pr_head::ensure_owner(&state)?;
1197        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1198            bail!(
1199                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1200            );
1201        }
1202        if let Some(to) = &state.released_to {
1203            bail!(
1204                "run {} cannot be resumed: its worktree was released to run {}",
1205                state.short(),
1206                crate::run::short_of(to)
1207            );
1208        }
1209        let roles = state.config.resolve_roles()?;
1210        if let Some(id) = &state.config.graph.selected_implementer
1211            && state.candidates.iter().any(|c| c.agent != *id)
1212        {
1213            bail!(
1214                "replacement implementer pin does not match the saved candidates; task must stay held"
1215            );
1216        }
1217        let max_parallel = state.config.graph.max_parallel.max(1);
1218        Ok(Self {
1219            state,
1220            roles,
1221            sem: Arc::new(Semaphore::new(max_parallel)),
1222            pause: Pause::new(),
1223            interrupt: Pause::new(),
1224        })
1225    }
1226
1227    /// Walk the graph to a terminal state, skipping nodes already recorded.
1228    ///
1229    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1230    /// the phone - ends here, so this is the one place a run that ended
1231    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1232    /// notification centre. Best-effort: see [`crate::notices::raise`].
1233    pub async fn execute(&mut self) -> Result<()> {
1234        if let Some(selection) = &self.state.pr_recovery
1235            && (selection.source_proof.is_some() || selection.source_run.is_some())
1236        {
1237            Box::pin(selection.verify_source(&self.state.repo)).await?;
1238        }
1239        // Queue claims cover daemon work; this also serializes manual drivers
1240        // consuming a repair answer, including the new review and decision.
1241        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1242            || self.state.land_approval.is_some()
1243            || self.state.merge_decision.is_some()
1244            || !self.state.merge_repairs.is_empty()
1245            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1246        {
1247            let claim = FixClaim::acquire(&self.state.dir())?;
1248            let disk = RunState::load(&self.state.id)?;
1249            let checkpoint = |s: &RunState| {
1250                (
1251                    s.merge_repairs.len(),
1252                    s.merge_repairs
1253                        .last()
1254                        .map(|r| (r.question.clone(), r.phase)),
1255                    s.land_approval
1256                        .as_ref()
1257                        .map(|a| (a.question.clone(), a.head.clone())),
1258                )
1259            };
1260            if checkpoint(&disk) != checkpoint(&self.state) {
1261                bail!(
1262                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1263                );
1264            }
1265            Some(claim)
1266        } else {
1267            None
1268        };
1269        let mut execution = self.claim_driver().await?;
1270        let runner = &mut *execution.runner;
1271        let wait = runner.repository_wait();
1272        // Keep the graph's large future off the callers' stack, including
1273        // standalone task attempts running on a default-sized test thread.
1274        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1275        let result = match result {
1276            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1277                record_repository_block(&mut runner.state, &e);
1278                runner.park_here()?;
1279                Ok(())
1280            }
1281            other => other,
1282        };
1283        execution.driver.finish(&mut runner.state).await?;
1284        let ended = if result.is_err() {
1285            Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1286        } else {
1287            crate::notices::run_ended(&runner.state)
1288        };
1289        if let Some(notice) = ended {
1290            crate::notices::raise(notice);
1291        }
1292        result
1293    }
1294
1295    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1296        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1297    }
1298
1299    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1300        let wait = self.repository_wait();
1301        coordinate_operation(&mut self.state, cwd, &wait).await
1302    }
1303
1304    /// Validate ownership and persist the driver claim in one critical section.
1305    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1306        let owner = match crate::pr_head::owner_lock_wait(
1307            &crate::run::home(),
1308            &self.state.id,
1309            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1310        )
1311        .await
1312        {
1313            Ok(owner) => owner,
1314            Err(error) => {
1315                let detail = format!("worktree maintenance or ownership update: {error:#}");
1316                // Admission did not own run.json: retain this resource outcome
1317                // in memory for queue settlement without overwriting a concurrent
1318                // retirement or publication transaction.
1319                self.state.resource_block = Some(crate::run::ResourceBlock {
1320                    node: "start".into(),
1321                    kind: "worktree maintenance".into(),
1322                    detail: detail.clone(),
1323                    at: Timestamp::now(),
1324                });
1325                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1326            }
1327        };
1328        let disk = RunState::load(&self.state.id)?;
1329        crate::pr_head::ensure_owner(&disk)?;
1330        if disk.liveness(false) == crate::run::Liveness::Live {
1331            bail!("run already has a live driver; wait for its execution to end");
1332        }
1333        if let Some(to) = &disk.released_to {
1334            bail!(
1335                "run {} cannot continue: its worktree was released to run {}",
1336                self.state.short(),
1337                crate::run::short_of(to)
1338            );
1339        }
1340        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1341        let mut execution = DriverExecution {
1342            runner: self,
1343            driver,
1344        };
1345        // Declare after the execution guard: on any claim error this short
1346        // lock must drop before that guard reloads and releases ownership.
1347        let _owner = owner;
1348        let state = &mut execution.runner.state;
1349        // Moving again, so it is no longer parked. Set before the walk rather
1350        // than in `resume`, so every way of re-entering the graph clears it
1351        // and a card cannot claim a run is waiting to be resumed while the
1352        // agents are already working.
1353        state.parked = false;
1354        state.resource_wait = None;
1355        state.resource_block = None;
1356        // Any seat this state still lists as answering belongs to whatever
1357        // process last drove this run — this one included, if it crashed
1358        // mid-wave. Cleared and flushed immediately, before anything else
1359        // runs, so a resume can never show a seat as live when nothing is
1360        // asking it anything yet; the node that actually dispatches the next
1361        // wave repopulates it.
1362        state.clear_active();
1363        // Recorded in the same spot, and flushed together with the clear
1364        // above: this is the pid a reader checks (`RunState::liveness`) when
1365        // no daemon claim exists to answer "is a process still driving this
1366        // run" — a plain `magi run` / `magi review` typed into a terminal
1367        // claims nothing there. Always overwritten, never only-if-absent, so
1368        // a resumed run's stale pid from a previous, possibly-dead process
1369        // can never survive into this one's own report. Unlike
1370        // `clear_active`, this changes on every single `execute()` call, so
1371        // the save below is now unconditional rather than only-if-cleared.
1372        //
1373        // `driver_started_at` is recorded in the same breath, from this same
1374        // pid, so `liveness` can tell a live pid that is genuinely still us
1375        // apart from one the OS has since handed to an unrelated process —
1376        // see that field's own doc for why the pid alone is not enough.
1377        let pid = std::process::id();
1378        state.driver_pid = Some(pid);
1379        state.driver_started_at = crate::proc::process_started_at(pid);
1380        state.driver_exited = false;
1381        state.begin_pass();
1382        execution.driver.bind(state);
1383        state.save()?;
1384        execution.driver.record(state)?;
1385        Ok(execution)
1386    }
1387
1388    async fn execute_graph(&mut self) -> Result<()> {
1389        self.state.configure_recovery_resume()?;
1390        // Preservation may invalidate an old empty classification. Resolve it
1391        // before any resume shortcut can reuse a verdict or publish a branch.
1392        self.persist_candidates().await?;
1393        if self
1394            .state
1395            .merge_decision
1396            .as_ref()
1397            .is_some_and(|decision| !decision.closed)
1398            && self.state.status != RunStatus::Landing
1399        {
1400            if crate::merge_decision::resumable_hold(&self.state) {
1401                self.state.merge = Some(MergeOutcome {
1402                    mode: MergeMode::Pr,
1403                    ok: false,
1404                    detail: self
1405                        .state
1406                        .merge_decision
1407                        .as_ref()
1408                        .unwrap()
1409                        .target
1410                        .pr
1411                        .clone(),
1412                    empty: false,
1413                });
1414                self.run_land().await?;
1415                self.settle_questions();
1416                return Ok(());
1417            }
1418            bail!(
1419                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1420            );
1421        }
1422        if self.state.config.graph.selected_implementer.is_some()
1423            && self.state.status == RunStatus::Blocked
1424            && self.state.tally.is_none()
1425            && self.state.candidate_preservation.is_empty()
1426            && self.state.interrupted_implementations.is_empty()
1427        {
1428            bail!(
1429                "selected implementation is held; explicitly release the owning task to retry it"
1430            );
1431        }
1432
1433        if self.state.merge_repairs.last().is_some_and(|r| {
1434            matches!(
1435                r.phase,
1436                crate::run::MergeRepairPhase::Requested
1437                    | crate::run::MergeRepairPhase::Fixing
1438                    | crate::run::MergeRepairPhase::Reviewing
1439            )
1440        }) {
1441            if self.repair_merge_review().await? {
1442                self.run_land().await?;
1443            }
1444            self.settle_questions();
1445            return Ok(());
1446        }
1447        // Publication already passed review and gate. Resume only that node:
1448        // syncing to a newer base here could change the verified commit.
1449        if self.state.pr_publication_pending {
1450            self.merge().await?;
1451            if self.resource_halt()? {
1452                return Ok(());
1453            }
1454            self.state.save()?;
1455            self.settle_questions();
1456            return Ok(());
1457        }
1458        // A run that already lost its quorum never resumes into the verdict
1459        // machinery: `deliberate` and `vote` would otherwise clobber the
1460        // stalled marker back to Voting and the run would keep going past a
1461        // verdict that is no longer trustworthy. Everything already recorded is
1462        // kept, so the run stays resumable (or foldable) for a human to pick up.
1463        //
1464        // On --resume the run gets one chance to repair itself: the seats a
1465        // rate limit took out are re-asked. If their quota has since reset and
1466        // the quorum is restored, the run picks up and finishes; otherwise it
1467        // stays stale and still-resumable for a later retry. If it does not
1468        // recover, the returned status stays `Stalled` and nothing was
1469        // clobbered (the recovery only mutates entries for the lost seats).
1470        if self.state.status == RunStatus::Stalled {
1471            if self.recover_stall().await? {
1472                self.finish_after_tally().await?;
1473            } else {
1474                // Still below quorum: persist the marker and stay resumable.
1475                self.state.save()?;
1476            }
1477            return Ok(());
1478        }
1479        // A run parked inside `land` - watching CI, mid fix-round, or
1480        // waiting on the owner's merge approval - resumes directly into it,
1481        // never back through `prep`. Everything before `merge` already
1482        // concluded; that is the only way `status` reaches `Landing` in the
1483        // first place. Re-walking `review_loop` first would also be actively
1484        // wrong: its own status recomputation (see its doc) treats any
1485        // clean round as reason to set `status` to `Gating`, which would
1486        // clobber this marker before `merge` ever ran, and this run would
1487        // never find its way back into `land` at all.
1488        if self.state.status == RunStatus::Landing
1489            || crate::handover::landing_checkpoint(&self.state)
1490        {
1491            self.state.land_sync_pending = true;
1492            self.state.save()?;
1493            self.run_land().await?;
1494            // `run_land` may have settled the run right here - CI came back
1495            // green and the PR merged, say - without ever passing back
1496            // through `merge`'s own trailing call. Whatever it left `status`
1497            // as is what this has to read.
1498            self.settle_questions();
1499            return Ok(());
1500        }
1501        // A run parked at `merge` on the owner's word about a withheld PR text
1502        // resumes straight into `merge`. Re-walking the tail would sync to the
1503        // base again (possibly rebasing, possibly spending the fixer) after the
1504        // gate had passed, and then push a tree nobody reviewed or gated.
1505        if self.state.github_text.is_some()
1506            && self.state.merge.is_none()
1507            && self.state.status == RunStatus::Gating
1508        {
1509            self.merge().await?;
1510            if self.resource_halt()? {
1511                return Ok(());
1512            }
1513            self.state.save()?;
1514            return Ok(());
1515        }
1516        if let Err(e) = self.prep().await {
1517            if e.is::<crate::repo_lock::ResourceBlocked>() {
1518                return Err(e);
1519            }
1520            self.state.status = RunStatus::Blocked;
1521            self.state
1522                .prep_failure
1523                .get_or_insert_with(|| format!("{e:#}"));
1524            self.state
1525                .event("prep", format!("preparation stopped: {e:#}"));
1526            self.state.save()?;
1527            return Err(e);
1528        }
1529        if self.park_here()? {
1530            return Ok(());
1531        }
1532        self.advise().await?;
1533        if self.park_here()? {
1534            return Ok(());
1535        }
1536        self.implement().await?;
1537        if self.park_here()? {
1538            return Ok(());
1539        }
1540        // `after_implement` already saved the state and settled any open
1541        // questions when it set this; nothing later in the graph has
1542        // anything to judge.
1543        if self.state.status == RunStatus::VerifiedNoop {
1544            return Ok(());
1545        }
1546        self.judge().await?;
1547        if self.park_here()? {
1548            return Ok(());
1549        }
1550        self.deliberate().await?;
1551        if self.park_here()? {
1552            return Ok(());
1553        }
1554        self.vote().await?;
1555        if self.park_here()? {
1556            return Ok(());
1557        }
1558        self.tally()?;
1559        // A verdict that lost its quorum is not trustworthy: do not review,
1560        // gate, or merge on it. Everything already done is kept, so the run
1561        // stays resumable (or foldable); the human can replace the agent that
1562        // ran out of quota and pick it up.
1563        if self.state.status == RunStatus::Stalled {
1564            // Persist the stalled marker now — the normal end-of-execute save
1565            // below is below this early return, and without it a resumed run
1566            // would reload a pre-tally status and keep going.
1567            self.state.save()?;
1568            return Ok(());
1569        }
1570        self.finish_after_tally().await?;
1571        Ok(())
1572    }
1573
1574    /// A wave could not start its agents because a shared resource stayed busy
1575    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1576    /// resumable. Unstarted seats have no result; completed peers retain
1577    /// their saved candidate state.
1578    fn stop_for_resource(&mut self) -> Result<()> {
1579        // A wait abandoned because the run is parking stays a park.
1580        if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1581        {
1582            self.state.status = RunStatus::Blocked;
1583        }
1584        self.state.save()
1585    }
1586
1587    /// Stop walking the graph if a node ended on a resource block.
1588    fn resource_halt(&mut self) -> Result<bool> {
1589        if self.state.resource_block.is_none() {
1590            return Ok(false);
1591        }
1592        self.stop_for_resource()?;
1593        Ok(true)
1594    }
1595
1596    /// Park here if asked to, recording it in the run's own timeline.
1597    ///
1598    /// Returns whether the caller should stop walking the graph. The state is
1599    /// saved either way by the node that just finished; this adds the event so
1600    /// the operator's card says why a run that is neither finished nor moving
1601    /// is sitting where it is.
1602    fn park_here(&mut self) -> Result<bool> {
1603        let parking = self.pause.parked() || self.interrupt.parked();
1604        if !parking && self.state.resource_block.is_some() {
1605            self.stop_for_resource()?;
1606            return Ok(true);
1607        }
1608        // Either handle asking is enough - see `Pause`'s own doc for why
1609        // they are never the same one. `interrupt` is checked second so a
1610        // reason it carries is preferred in the message below over a plain
1611        // shutdown park racing it at the same boundary.
1612        if !self.pause.parked() && !self.interrupt.parked() {
1613            return Ok(false);
1614        }
1615        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1616        let why = match cause.as_ref().map(ParkCause::label) {
1617            Some(reason) => format!(
1618                "parked after `{}` ({reason}) — resume to carry on from here",
1619                self.state.status.as_str()
1620            ),
1621            None => format!(
1622                "parked after `{}` — resume to carry on from here",
1623                self.state.status.as_str()
1624            ),
1625        };
1626        self.state.event("park", why);
1627        self.state
1628            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1629        self.state.parked = true;
1630        self.state.save()?;
1631        Ok(true)
1632    }
1633
1634    /// Only a new explicit operator approval may retry failed preparation.
1635    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1636        if self.state.prep_failure.is_none() {
1637            return Ok(());
1638        }
1639        let failed_at = self
1640            .state
1641            .events
1642            .iter()
1643            .rev()
1644            .find(|event| event.node == "prep")
1645            .map_or(self.state.updated_at, |event| event.at);
1646        anyhow::ensure!(
1647            approval_at > failed_at,
1648            "preparation retry requires a new explicit release after the failure"
1649        );
1650        self.state.prep_failure = None;
1651        self.state.prep_complete = Some(false);
1652        self.state
1653            .event("prep-retry", "operator approved retry of preparation");
1654        Ok(())
1655    }
1656
1657    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1658    pub fn on_pause(&mut self, pause: Pause) {
1659        self.pause = pause;
1660    }
1661
1662    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1663    /// scheduler asking this one run - and no other - to park so a task
1664    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1665    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1666    /// handle.
1667    pub fn watch_interrupt(&mut self, pause: Pause) {
1668        self.interrupt = pause;
1669    }
1670
1671    /// Abandon this run's own open questions, once `status` has actually
1672    /// settled rather than merely paused.
1673    ///
1674    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1675    /// either back up with the candidates, the review round and the seat
1676    /// sessions already on disk, so a question an implementer asked mid-round
1677    /// may still get a real answer read by a real resume. Only the statuses
1678    /// `resumable` excludes are actually final: the run merged, it reached
1679    /// `Ready` with nothing left to do, it failed outright with no
1680    /// established point to continue from, or every candidate agreed, with
1681    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1682    /// every one of those the seat that asked is gone for good, exactly like
1683    /// the run being deleted under `magi run rm` - so the same cleanup
1684    /// applies, worded for what actually happened instead of "the run was
1685    /// deleted".
1686    ///
1687    /// Best-effort and silent on success: called from every place `status`
1688    /// can land on one of those three, including ones a resumed run revisits,
1689    /// so it must cost nothing when there was nothing open to begin with.
1690    fn settle_questions(&mut self) {
1691        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1692            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1693        }
1694    }
1695
1696    /// The tail of the graph after a trustworthy tally: fold losers, review,
1697    /// gate, merge, and persist.
1698    async fn finish_after_tally(&mut self) -> Result<()> {
1699        // Before the losers are folded: their branches are what the graft
1700        // reads from.
1701        self.graft().await?;
1702        if self.resource_halt()? {
1703            return Ok(());
1704        }
1705        self.fold_losers().await?;
1706        // Before review starts, and again right before the gate: a run's
1707        // review rounds can themselves take long enough for the base to move
1708        // a second time, and the gate is the one node whose "green" gets
1709        // acted on.
1710        self.sync_to_base().await?;
1711        if self.resource_halt()? {
1712            return Ok(());
1713        }
1714        if self.state.status == RunStatus::AlreadyInBase {
1715            return Ok(());
1716        }
1717        self.review_loop().await?;
1718        if self.resource_halt()? {
1719            return Ok(());
1720        }
1721        self.sync_to_base().await?;
1722        if self.resource_halt()? {
1723            return Ok(());
1724        }
1725        if self.state.status == RunStatus::AlreadyInBase {
1726            return Ok(());
1727        }
1728        self.gate().await?;
1729        self.merge().await?;
1730        if self.resource_halt()? {
1731            return Ok(());
1732        }
1733        self.state.save()?;
1734        Ok(())
1735    }
1736
1737    // ---------------------------------------------------------------- prep
1738
1739    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1740    /// run with the step and its output in the event; it is never swallowed.
1741    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1742    /// resume starts over instead of finding candidates and missing seats.
1743    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1744        let result = match ensure_setup_config(&mut self.state, repo).await {
1745            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1746            Err(e) => Err(e),
1747        };
1748        if let Err(e) = result {
1749            git::worktree_remove_clean(repo, wt).await.ok();
1750            if in_prep {
1751                self.rollback_prep(repo).await;
1752            }
1753            self.state.status = RunStatus::Blocked;
1754            self.state.event(
1755                "setup",
1756                format!("worktree setup failed in {}: {e:#}", wt.display()),
1757            );
1758            self.state.save()?;
1759            return Err(e);
1760        }
1761        Ok(())
1762    }
1763
1764    /// Undo everything `prep` created, so a resume re-runs it whole.
1765    async fn rollback_prep(&mut self, repo: &Path) {
1766        let root = self.state.worktree_root();
1767        for c in std::mem::take(&mut self.state.candidates) {
1768            if matches!(
1769                git::worktree_remove_clean(repo, &c.worktree).await,
1770                Ok(true)
1771            ) {
1772                git::branch_delete(repo, &c.branch).await.ok();
1773            } else {
1774                self.state.candidates.push(c);
1775            }
1776        }
1777        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1778        for j in 1..=seats {
1779            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1780                .await
1781                .ok();
1782        }
1783        for k in 1..=self.state.config.graph.advisors {
1784            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1785                .await
1786                .ok();
1787        }
1788    }
1789
1790    async fn prep(&mut self) -> Result<()> {
1791        if let Some(reason) = &self.state.prep_failure {
1792            bail!(
1793                "candidate preparation previously failed: {reason}; correct the source before retrying"
1794            );
1795        }
1796        if self
1797            .state
1798            .prep_complete
1799            .unwrap_or(!self.state.candidates.is_empty())
1800        {
1801            return Ok(());
1802        }
1803        let repo = self.state.repo.clone();
1804        // Includes rollback, config, refs, checkout creation and setup. No
1805        // repository mutation may precede the real shared lock acquisition.
1806        let _coordination = self.coordinate(&repo).await?;
1807        let base = self.state.base_commit.clone();
1808        let plan = self.state.sources.plan(&repo, &base).await?;
1809        self.rollback_prep(&repo).await;
1810        if !self.state.candidates.is_empty() {
1811            bail!(
1812                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1813            );
1814        }
1815        self.state.prep_complete = Some(false);
1816        self.state.status = RunStatus::Prep;
1817        self.state.save()?;
1818        let repo = self.state.repo.clone();
1819        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1820        let root = self.state.worktree_root();
1821        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1822
1823        // The hook is the write-time half of the blindness contract; the
1824        // presentation filter in `blind` is the half that cannot be bypassed.
1825        let hooks_dir = self.state.dir().join("hooks");
1826        if self.state.config.blind.commit_msg_hook {
1827            std::fs::create_dir_all(&hooks_dir)
1828                .with_context(|| format!("create {}", hooks_dir.display()))?;
1829            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1830            let path = hooks_dir.join("commit-msg");
1831            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1832            make_executable(&path)?;
1833            // Ref-counted rather than a plain idempotent set: with more than
1834            // one run able to be in flight in the same repository at once
1835            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1836            // true?" check cannot tell "another run of mine still needs
1837            // this" from "nobody does", and the run that happens to finish
1838            // first would disable the hook out from under a sibling still
1839            // relying on it.
1840            git::acquire_worktree_config(&repo).await?;
1841            self.state.enabled_worktree_config = true;
1842        }
1843
1844        // `[worktree] setup` hides its products per worktree, which needs the
1845        // same per-worktree config the hook does. Held until fold, like it.
1846        ensure_setup_config(&mut self.state, &repo).await?;
1847
1848        for (index, (spec, label)) in self
1849            .roles
1850            .implementers
1851            .clone()
1852            .into_iter()
1853            .zip(labels)
1854            .enumerate()
1855        {
1856            let branch = self.state.branch_for(label);
1857            let worktree = root.join(format!("cand-{label}"));
1858            // Persist intent before Git can leave a partially created tree.
1859            // Candidate registration follows setup, so it cannot inventory
1860            // crashes or refusals during creation itself.
1861            self.state.event(
1862                "prep-intent",
1863                serde_json::json!({
1864                    "branch": branch,
1865                    "worktree": worktree,
1866                    "start": start,
1867                    "picks": plan.picks,
1868                })
1869                .to_string(),
1870            );
1871            self.state.save()?;
1872            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1873            if self.state.config.blind.commit_msg_hook {
1874                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1875            }
1876            git::local_exclude(&worktree, "/.magi/").await?;
1877            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1878                // Nothing of this prep may survive: a resume must start over,
1879                // not find half the candidates and skip the rest.
1880                git::worktree_remove_clean(&repo, &worktree).await.ok();
1881                git::branch_delete(&repo, &branch).await.ok();
1882                self.rollback_prep(&repo).await;
1883                self.state.status = RunStatus::Blocked;
1884                self.state
1885                    .event("prep", format!("worktree setup failed: {e:#}"));
1886                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1887                self.state.save()?;
1888                return Err(e);
1889            }
1890            self.state.candidates.push(Candidate {
1891                index,
1892                label,
1893                agent: spec.id.clone(),
1894                branch,
1895                worktree,
1896                summary: String::new(),
1897                stat: String::new(),
1898                files: 0,
1899                commits: 0,
1900                empty: false,
1901                failed: None,
1902                verified_noop: None,
1903                duration_ms: 0,
1904                folded: false,
1905            });
1906            self.state.save()?;
1907            for pick in &plan.picks {
1908                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1909                {
1910                    self.state.status = RunStatus::Blocked;
1911                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1912                    self.state
1913                        .event("prep", format!("cannot apply referenced commit: {e}"));
1914                    self.state.save()?;
1915                    return Err(e);
1916                }
1917            }
1918        }
1919
1920        for j in 1..=self.roles.judges.len() {
1921            let wt = root.join(format!("judge-{j}"));
1922            if !wt.exists() {
1923                git::worktree_add_detached(&repo, &wt, &base).await?;
1924                self.setup_seat_worktree(&repo, &wt, true).await?;
1925            }
1926        }
1927
1928        // Disposable, detached checkouts for the design-deliberation stage's
1929        // advisor seats — the same shape as the judges' above, at the same
1930        // base commit, since advisors also only ever read. Sized off the
1931        // configured count directly rather than a resolved roster: unlike
1932        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1933        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1934        // `prep` has no `ResolvedRoles` field to read a count from here.
1935        if self.state.config.graph.advise {
1936            for k in 1..=self.state.config.graph.advisors {
1937                let wt = root.join(format!("advisor-{k}"));
1938                if !wt.exists() {
1939                    git::worktree_add_detached(&repo, &wt, &base).await?;
1940                    self.setup_seat_worktree(&repo, &wt, true).await?;
1941                }
1942            }
1943        }
1944
1945        // A judge cannot tell it is looking at its own patch — the seats keep
1946        // separate conversations — but a panel that shares agents with the
1947        // field is less independent than it looks, and that is worth saying out
1948        // loud once per run rather than leaving it in the config.
1949        let authors: Vec<&str> = self
1950            .roles
1951            .implementers
1952            .iter()
1953            .map(|a| a.id.as_str())
1954            .collect();
1955        let overlap: Vec<String> = self
1956            .roles
1957            .judges
1958            .iter()
1959            .enumerate()
1960            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1961            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1962            .collect();
1963        if !overlap.is_empty() {
1964            let note = format!(
1965                "{} also authored a candidate; blind, but the panel is less \
1966                 independent than {} distinct agents would be",
1967                overlap.join(", "),
1968                self.roles.judges.len()
1969            );
1970            self.state.event("prep", note);
1971        }
1972
1973        self.state.event(
1974            "prep",
1975            format!(
1976                "{} candidates, {} judges, base {} ({})",
1977                self.state.candidates.len(),
1978                self.roles.judges.len(),
1979                &self.state.base_commit[..7.min(self.state.base_commit.len())],
1980                self.state.base_branch
1981            ),
1982        );
1983        self.state.prep_complete = Some(true);
1984        self.state.status = RunStatus::Implementing;
1985        self.state.save()?;
1986        Ok(())
1987    }
1988
1989    // -------------------------------------------------------------- advise
1990
1991    /// The design-deliberation stage: independent, read-only advisor seats
1992    /// each sketch a design before any implementer touches the repository,
1993    /// and (when at least one produced a usable proposal) a synthesis seat
1994    /// blends them into a brief `implement` carries in every candidate's
1995    /// prompt.
1996    ///
1997    /// `[graph] advise` is the on/off switch, on by default; `[graph]
1998    /// advisors` is the proposal count. Everything here is best-effort and
1999    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2000    /// that cannot reach quota, or a synthesis seat that produced nothing
2001    /// usable all leave `implement` exactly as it was before this stage
2002    /// existed — the task instruction alone — rather than failing the whole
2003    /// competition over an enrichment stage. Every outcome is still recorded
2004    /// as an event, so a run that got nothing from this stage says why.
2005    ///
2006    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2007    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2008    /// resumed run whose stage failed would re-run it, and re-spend the
2009    /// agent calls, on every reentry before `implement`.
2010    ///
2011    /// Also skipped once any candidate shows implementation progress — the
2012    /// exact predicate `implement` itself uses to decide a candidate is no
2013    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2014    /// is not enough: a run created by an older binary that predates this
2015    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2016    /// an already-`Implementing`-or-later run under this build would
2017    /// otherwise walk straight back through `prep` (a no-op once candidates
2018    /// exist) into this node and spawn every advisor seat against worktrees
2019    /// `prep` never recreated — after implementation has already started,
2020    /// which is exactly the invariant this stage exists to guarantee.
2021    async fn advise(&mut self) -> Result<()> {
2022        let implement_untouched = self
2023            .state
2024            .candidates
2025            .iter()
2026            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2027        if !self.state.config.graph.advise || self.state.advise_attempted {
2028            return Ok(());
2029        }
2030        if !implement_untouched {
2031            self.state.event(
2032                "advise",
2033                "skipping the design-deliberation stage: at least one \
2034                 candidate already shows implementation progress, so this \
2035                 run is past the point the stage exists to run before"
2036                    .to_owned(),
2037            );
2038            self.state.advise_attempted = true;
2039            self.state.save()?;
2040            return Ok(());
2041        }
2042        let run_id = self.state.id.clone();
2043        let prompts = self.state.config.prompts.clone();
2044        let instruction = self.state.instruction.clone();
2045        let language = self.state.config.graph.language.clone();
2046        let root = self.state.worktree_root();
2047        let n = self.state.config.graph.advisors;
2048        let where_recorded = self.state.dir().join("run.json");
2049
2050        let seats = match self.state.config.advisors() {
2051            Ok(seats) if !seats.is_empty() => seats,
2052            Ok(_) => {
2053                self.state.event(
2054                    "advise",
2055                    format!(
2056                        "[graph] advisors is 0; skipping the design-deliberation \
2057                         stage and continuing without a synthesis brief (see {})",
2058                        where_recorded.display()
2059                    ),
2060                );
2061                self.state.advise_attempted = true;
2062                self.state.save()?;
2063                return Ok(());
2064            }
2065            Err(e) => {
2066                self.state.event(
2067                    "advise",
2068                    format!(
2069                        "could not resolve advisor seats ({e:#}); continuing \
2070                         without a design-deliberation brief (see {})",
2071                        where_recorded.display()
2072                    ),
2073                );
2074                self.state.advise_attempted = true;
2075                self.state.save()?;
2076                return Ok(());
2077            }
2078        };
2079
2080        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2081        let artifacts = agent::artifacts_dir(&self.state.dir());
2082        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2083
2084        let mut jobs = Vec::new();
2085        for (i, spec) in seats.iter().cloned().enumerate() {
2086            let seat_key = format!("advisor-{}", i + 1);
2087            let seat = self.seat(&seat_key, &spec.id);
2088            jobs.push(SeatJob {
2089                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2090                spec,
2091                seat,
2092                cwd: worktrees[i % worktrees.len()].clone(),
2093                timeout,
2094                allow_write: false,
2095                sessions: false,
2096                artifacts: artifacts.clone(),
2097                stem: seat_key,
2098                handover: None,
2099            });
2100        }
2101
2102        self.state.event(
2103            "advise",
2104            format!(
2105                "{} advisor seat(s) sketching a design in parallel",
2106                jobs.len()
2107            ),
2108        );
2109        let mut quota_losses = Vec::new();
2110        let cache = self.state.config.cache_dir();
2111        let ctx = WaveCtx {
2112            carry_seats: false,
2113            pauses: Some((&self.pause, &self.interrupt)),
2114            run: &run_id,
2115            node: "advise",
2116            prompts: &prompts,
2117            cache: cache.as_deref(),
2118            round: None,
2119        };
2120        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2121        let results = ask_json_wave::<Proposal>(
2122            jobs,
2123            Arc::clone(&self.sem),
2124            self.state.config.graph.retries,
2125            &advisor_roster,
2126            &ctx,
2127            &mut quota_losses,
2128            &mut self.state,
2129            &|p: &Proposal| p.validate(),
2130        )
2131        .await;
2132        if self.state.resource_block.is_some() {
2133            return self.stop_for_resource();
2134        }
2135        self.state.quota.extend(quota_losses);
2136
2137        let mut records = Vec::with_capacity(results.len());
2138        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2139            let agent_id = seat.agent.clone();
2140            self.state.seats.insert(seat.key.clone(), seat);
2141            match res {
2142                Ok((proposal, out)) => {
2143                    self.state
2144                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2145                    records.push(advise::AdvisorRecord::proposed(
2146                        i + 1,
2147                        agent_id,
2148                        proposal,
2149                        out.duration_ms,
2150                    ));
2151                }
2152                Err(e) => {
2153                    self.state.event(
2154                        "advise",
2155                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2156                    );
2157                    records.push(advise::AdvisorRecord::failed(
2158                        i + 1,
2159                        agent_id,
2160                        e.to_string(),
2161                    ));
2162                }
2163            }
2164        }
2165
2166        let mut advice = advise::Advice {
2167            records,
2168            synthesis: None,
2169        };
2170        if advice.proposals().is_empty() {
2171            self.state.event(
2172                "advise",
2173                "no advisor produced a usable proposal; continuing without a \
2174                 synthesis brief"
2175                    .to_owned(),
2176            );
2177        } else {
2178            match self
2179                .synthesize_brief(
2180                    &advice,
2181                    &instruction,
2182                    &language,
2183                    &worktrees[0],
2184                    &artifacts,
2185                    &run_id,
2186                    &prompts,
2187                    cache.as_deref(),
2188                )
2189                .await
2190            {
2191                Ok(Some(text)) => {
2192                    self.state.event(
2193                        "advise",
2194                        "synthesized a design brief for the implementer".to_owned(),
2195                    );
2196                    advice.synthesis = Some(text);
2197                }
2198                Ok(None) => {
2199                    self.state.event(
2200                        "advise",
2201                        "the synthesis seat produced nothing usable; continuing \
2202                         without a design brief"
2203                            .to_owned(),
2204                    );
2205                }
2206                Err(e) => {
2207                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2208                        return Err(e);
2209                    }
2210                    self.state.event(
2211                        "advise",
2212                        format!("could not synthesize a design brief: {e:#}"),
2213                    );
2214                }
2215            }
2216        }
2217        advise::apply_reflection(&mut advice);
2218
2219        self.state.advice = Some(advice);
2220        self.state.advise_attempted = true;
2221        self.state.save()?;
2222        Ok(())
2223    }
2224
2225    /// The synthesis seat: reads every advisor's proposal and blends them
2226    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2227    /// out of [`Runner::advise`] only for readability — it is not called
2228    /// anywhere else.
2229    ///
2230    /// Picked the same way [`crate::talk`]'s standing conversation and
2231    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2232    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2233    /// order (a claude seat, else the first runnable agent in roster order)
2234    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2235    /// in [`crate::config`] for why a dedicated field exists here at all.
2236    #[allow(clippy::too_many_arguments)]
2237    async fn synthesize_brief(
2238        &mut self,
2239        advice: &advise::Advice,
2240        instruction: &str,
2241        language: &str,
2242        cwd: &Path,
2243        artifacts: &Path,
2244        run_id: &str,
2245        prompts: &Prompts,
2246        cache: Option<&Path>,
2247    ) -> Result<Option<String>> {
2248        let chain = agent::pick_chain(
2249            &self.state.config.agents,
2250            self.state.config.roles.synthesizer.as_ref(),
2251            &agent::installed,
2252            "synthesizer",
2253        )?;
2254        let proposals = advice.proposals();
2255        let mut prompt = prompt::with_overlay(
2256            prompt::synthesize_brief(instruction, &proposals, language),
2257            prompts.overlay("advise"),
2258        );
2259        if cache.is_some() {
2260            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2261            // below — see `prompt::build_cache_note`'s doc for why telling a
2262            // read-only seat to build through the shared cache is exactly how
2263            // a sandbox's write refusal gets misread as a defect.
2264            prompt.push('\n');
2265            prompt.push_str(&prompt::build_cache_note("advise", false));
2266        }
2267        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2268        // Each id is tried once, in order; a quota hit, error or unusable
2269        // answer moves to the next. The seat is single-turn (`sessions:
2270        // false`) and the prompt is the whole context, so a fallback agent
2271        // needs nothing carried over.
2272        let mut last = None;
2273        for (n, spec) in chain.iter().enumerate() {
2274            if n > 0 {
2275                self.state
2276                    .event("advise", format!("synthesis falling back to {}", spec.id));
2277            }
2278            let mut seat = self.seat("advise-synthesis", &spec.id);
2279            let wait = self.repository_wait();
2280            let outcome = wait
2281                .scope(agent::invoke(
2282                    spec,
2283                    &mut seat,
2284                    &Invocation {
2285                        scope: crate::agent::InvocationScope::Repository,
2286                        coordination: self.state.config.coordination.as_ref(),
2287                        fetched_base: None,
2288                        cwd,
2289                        prompt: &prompt,
2290                        timeout,
2291                        allow_write: false,
2292                        unsandboxed: false,
2293                        sessions: false,
2294                        artifacts,
2295                        stem: &if n == 0 {
2296                            "advise-synthesis".to_owned()
2297                        } else {
2298                            format!("advise-synthesis-{}", spec.id)
2299                        },
2300                        run: run_id,
2301                        node: "advise",
2302                        cache_dir: None,
2303                        attachments: &[],
2304                        writable: &[],
2305                    },
2306                ))
2307                .await;
2308            if outcome.is_ok() {
2309                self.state.seats.insert(seat.key.clone(), seat);
2310            }
2311            let advance = agent::chain_advances(&outcome);
2312            last = Some(outcome);
2313            if !advance {
2314                break;
2315            }
2316        }
2317        // Exhausted: the last attempt's result is what a single failed seat
2318        // would have produced.
2319        let out = last.expect("a chain holds at least one agent")?;
2320        if !out.usable() {
2321            return Ok(None);
2322        }
2323        let text =
2324            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2325        Ok((!text.trim().is_empty()).then_some(text))
2326    }
2327
2328    // ----------------------------------------------------------- implement
2329
2330    async fn implement(&mut self) -> Result<()> {
2331        self.persist_candidates().await?;
2332        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2333        // agent files with `magi task add` name the run that paid for it. The
2334        // prompt overlay is cloned alongside it because the waves borrow it
2335        // while `self` is mutably borrowed by the node's own bookkeeping.
2336        let run_id = self.state.id.clone();
2337        let prompts = self.state.config.prompts.clone();
2338        let todo: Vec<usize> = self
2339            .state
2340            .candidates
2341            .iter()
2342            .enumerate()
2343            .filter(|(_, c)| {
2344                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2345                    && c.failed.is_none()
2346                    && !c.empty
2347            })
2348            .map(|(i, _)| i)
2349            .collect();
2350        if todo.is_empty() {
2351            return self.after_implement();
2352        }
2353        self.state.status = RunStatus::Implementing;
2354
2355        let language = self.state.config.graph.language.clone();
2356        let github_language = self.state.config.graph.github_language.clone();
2357        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2358        let sessions = self.state.config.graph.sessions;
2359        let artifacts = agent::artifacts_dir(&self.state.dir());
2360        // The design-deliberation stage's blended brief, when `advise` found
2361        // one — carried into every implementer's prompt the same way
2362        // regardless of which candidate it is.
2363        let brief = self
2364            .state
2365            .advice
2366            .as_ref()
2367            .and_then(|a| a.synthesis.as_deref())
2368            .map(str::to_owned);
2369        let attachments = self.state.attachments.clone();
2370
2371        let mut jobs = Vec::new();
2372        for &i in &todo {
2373            let (index, label, worktree) = {
2374                let c = &self.state.candidates[i];
2375                (c.index, c.label, c.worktree.clone())
2376            };
2377            let spec = self.roles.implementers[index].clone();
2378            let seat_key = format!("impl-{label}");
2379            let seat = self.seat(&seat_key, &spec.id);
2380            let instruction = seeded_instruction(&self.state);
2381            jobs.push(SeatJob {
2382                spec,
2383                seat,
2384                prompt: prompt::implement_in(
2385                    &instruction,
2386                    &worktree.to_string_lossy(),
2387                    &language,
2388                    &github_language,
2389                    brief.as_deref(),
2390                    &attachments,
2391                ),
2392                cwd: worktree,
2393                timeout,
2394                allow_write: true,
2395                sessions,
2396                artifacts: artifacts.clone(),
2397                stem: format!("impl-{label}"),
2398                handover: None,
2399            });
2400        }
2401
2402        self.state.event(
2403            "implement",
2404            format!("{} candidates in parallel", jobs.len()),
2405        );
2406        // Kept so a seat whose CLI hung up can be asked again from the same
2407        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2408        // can update a seat's own entry once a fallback agent takes it over —
2409        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2410        // whichever agent actually answered, not the one that quota'd out.
2411        let mut sent = jobs.clone();
2412        let cache = self.state.config.cache_dir();
2413        let ctx = WaveCtx {
2414            carry_seats: false,
2415            pauses: Some((&self.pause, &self.interrupt)),
2416            run: &run_id,
2417            node: "implement",
2418            prompts: &prompts,
2419            cache: cache.as_deref(),
2420            round: None,
2421        };
2422        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2423        if !wave_blocked(&results) {
2424            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2425                .await;
2426            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2427                .await?;
2428            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2429                .await;
2430        }
2431        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2432
2433        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2434            && results
2435                .iter()
2436                .any(|(_, _, out)| FailClass::of(out).is_some());
2437        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2438            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2439                // Executed turns interrupted by disk pressure already carry
2440                // a preservation barrier from wave. Admission-only waits do
2441                // not. Keep the interrupted seat's session for recovery too.
2442                if self
2443                    .state
2444                    .candidate_preservation
2445                    .contains_key(&self.state.candidates[i].label)
2446                {
2447                    self.state.seats.insert(seat.key.clone(), seat);
2448                }
2449                continue;
2450            }
2451            let seat_key = seat.key.clone();
2452            // A quota fallback (`resume_seat_handovers`) may have handed this
2453            // seat to a different agent than the one `prep` recorded on the
2454            // candidate; the stats tables and any later fixer-defaults-to-
2455            // winner's-author lookup must credit whoever actually answered —
2456            // unless every fallback also quota'd out, in which case nobody
2457            // actually answered and crediting the last agent tried would
2458            // erase every earlier agent's own quota loss from the stats
2459            // tables instead of just this one seat's.
2460            let agent = seat.agent.clone();
2461            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2462            self.state.seats.insert(seat.key.clone(), seat);
2463            let label = self.state.candidates[i].label;
2464            self.state.interrupted_implementations.remove(&label);
2465
2466            let (summary, duration, failed, verified_claim) = match out {
2467                AgentOutcome::Ok(o) => {
2468                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2469                    let failed = (!o.usable()).then(|| {
2470                        if o.timed_out {
2471                            "agent timed out".to_owned()
2472                        } else {
2473                            format!(
2474                                "agent exited with {}",
2475                                crate::notices::exit_status(o.exit_code)
2476                            )
2477                        }
2478                    });
2479                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2480                    (text, o.duration_ms, failed, verified_claim)
2481                }
2482                // Left un-resumed by `resume_undelivered` (a dirty tree
2483                // already rescues the work, or there was no session left to
2484                // resume into) — reported like the ordinary failure it is,
2485                // never as if `o.text` (the CLI's raw error JSON) were an
2486                // answer.
2487                AgentOutcome::Dropped(o) => {
2488                    let why = o
2489                        .dropped
2490                        .as_ref()
2491                        .map(|d| d.why.as_str())
2492                        .unwrap_or("the CLI ended the stream without delivering its answer");
2493                    (
2494                        String::new(),
2495                        o.duration_ms,
2496                        Some(format!("the CLI dropped the stream ({why})")),
2497                        None,
2498                    )
2499                }
2500                AgentOutcome::Quota(o) => {
2501                    self.state.quota.push(QuotaLoss {
2502                        seat: seat_key,
2503                        node: "implement".to_owned(),
2504                        at: Timestamp::now(),
2505                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2506                    });
2507                    (
2508                        String::new(),
2509                        o.duration_ms,
2510                        Some("rate limited (quota); produced no change".to_owned()),
2511                        None,
2512                    )
2513                }
2514                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2515                    (String::new(), 0, Some(e), None)
2516                }
2517            };
2518
2519            // Save every completed turn before any fallible Git operation. A failed
2520            // rescue must not erase the reply or prevent collection of other seats.
2521            let c = &mut self.state.candidates[i];
2522            if !exhausted_the_fallback_chain {
2523                c.agent = agent;
2524            }
2525            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2526            c.duration_ms = duration;
2527            c.failed = failed;
2528            c.verified_noop = verified_claim;
2529            self.state
2530                .candidate_preservation
2531                .insert(label, "awaiting persistence".to_owned());
2532        }
2533        self.state.save()?;
2534        self.persist_candidates().await?;
2535
2536        if resource_blocked {
2537            return self.stop_for_resource();
2538        }
2539        if selected_failed {
2540            self.state.status = RunStatus::Blocked;
2541            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2542            self.state.save()?;
2543            bail!(
2544                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2545            );
2546        }
2547        self.after_implement()
2548    }
2549
2550    /// Retry only persistence after a completed implementation turn.
2551    async fn persist_candidates(&mut self) -> Result<()> {
2552        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2553        for label in labels {
2554            let i = self
2555                .state
2556                .candidates
2557                .iter()
2558                .position(|c| c.label == label)
2559                .context("pending candidate is missing")?;
2560            let c = self.state.candidates[i].clone();
2561            let interrupted = self.state.interrupted_implementations.contains(&label);
2562            let result = async {
2563                let wait = crate::repo_lock::WaitPolicy::new(
2564                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2565                    || false,
2566                );
2567                let _coordination =
2568                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2569                // Best effort evidence requires no index writes. A successful snapshot
2570                // never authorizes removal of a tree whose rescue failed.
2571                if let Err(e) = git::snapshot_uncommitted(
2572                    &c.worktree,
2573                    &self
2574                        .state
2575                        .dir()
2576                        .join("artifacts")
2577                        .join(format!("candidate-{label}-recovery")),
2578                )
2579                .await
2580                {
2581                    self.state.event(
2582                        "preserve",
2583                        format!("candidate {label} snapshot incomplete: {e:#}"),
2584                    );
2585                }
2586                let rescued = git::rescue_commit(
2587                    &c.worktree,
2588                    &format!("magi: candidate {label} (uncommitted work)"),
2589                )
2590                .await?;
2591                self.state.note_withheld("implement", &rescued.withheld);
2592                let base = &self.state.base_commit;
2593                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2594                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2595                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2596                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2597                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2598                if c.empty && commits > 0 && !patch.trim().is_empty()
2599                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2600                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2601                        || self.state.tally.is_some())
2602                {
2603                    // Keep the old classification and pending marker so every
2604                    // retry stops too. Retain decisions as recovery evidence;
2605                    // only a supported recovery operation may replace them.
2606                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2607                }
2608                let c = &mut self.state.candidates[i];
2609                c.commits = commits;
2610                c.stat = stat;
2611                c.files = files;
2612                let no_changes = commits == 0 || patch.trim().is_empty();
2613                c.empty = no_changes && !interrupted;
2614                if interrupted && no_changes {
2615                    c.failed = None;
2616                    c.verified_noop = None;
2617                }
2618                if !c.empty {
2619                    c.failed = None;
2620                    c.verified_noop = None;
2621                }
2622                self.state.event(
2623                    "preserve",
2624                    format!("candidate {label}: {files} files, {commits} commits"),
2625                );
2626                Ok::<_, anyhow::Error>(())
2627            }
2628            .await;
2629            match result {
2630                Ok(()) => {
2631                    self.state.candidate_preservation.remove(&label);
2632                    if !interrupted || self.state.candidates[i].files > 0 {
2633                        self.state.interrupted_implementations.remove(&label);
2634                    }
2635                }
2636                Err(e) => {
2637                    let reason = format!("candidate {label} preservation failed: {e:#}");
2638                    self.state.event("preserve", &reason);
2639                    self.state.candidate_preservation.insert(label, reason);
2640                }
2641            }
2642            if self.state.candidate_preservation.is_empty()
2643                && self.state.status == RunStatus::Blocked
2644                && self.state.candidates.iter().all(|c| c.failed.is_none())
2645            {
2646                // Persist the recovered phase with the cleared pending marker;
2647                // a crash between saves must not turn this into a selected-seat
2648                // failure hold on the next resume.
2649                self.state.status = RunStatus::Implementing;
2650                self.state.event(
2651                    "preserve",
2652                    "candidate persistence recovered; implementation can continue",
2653                );
2654            }
2655            self.state.save()?;
2656        }
2657        if !self.state.candidate_preservation.is_empty() {
2658            self.state.status = RunStatus::Blocked;
2659            self.state.save()?;
2660            bail!(
2661                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2662            );
2663        }
2664        Ok(())
2665    }
2666
2667    /// Ask again, once, for work a CLI did and then failed to hand over.
2668    ///
2669    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2670    /// status with an empty response and a usage report showing output tokens,
2671    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2672    /// seven minutes and 14,267 output tokens that arrived as an empty
2673    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2674    ///
2675    /// Two conditions, and both matter:
2676    ///
2677    /// - **Only when the tree is untouched.** Often the agent has already
2678    ///   written its files and only the closing message was lost; the rescue
2679    ///   commit below picks that up and there is nothing to ask for. Re-asking
2680    ///   then would pay for a second implementation of work already on disk.
2681    /// - **Once.** A CLI that drops one stream can drop the next, and this
2682    ///   node is the most expensive in the graph.
2683    ///
2684    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2685    /// dropped reply still carried its `conversation_id`, so the seat is asked
2686    /// to finish what it was doing rather than sent the whole task again. It
2687    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2688    /// node's - for the same reason a re-ranked judge does: restating finished
2689    /// work is not the work.
2690    ///
2691    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2692    /// same way until it resets, while an abandoned conversation is still
2693    /// there to be picked up.
2694    async fn resume_undelivered(
2695        &mut self,
2696        results: &mut [(usize, SeatState, AgentOutcome)],
2697        sent: &[SeatJob],
2698        prompts: &Prompts,
2699        run_id: &str,
2700    ) {
2701        for (wi, seat, out) in results.iter_mut() {
2702            let Some(dropped) = (match &*out {
2703                AgentOutcome::Dropped(o) => o.dropped.clone(),
2704                _ => None,
2705            }) else {
2706                continue;
2707            };
2708            let Some(job) = sent.get(*wi) else { continue };
2709            // Already on disk? Then only the closing message was lost.
2710            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2711                self.state.event(
2712                    "implement",
2713                    format!(
2714                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2715                         work is in the tree",
2716                        seat.key, dropped.output_tokens, dropped.why
2717                    ),
2718                );
2719                continue;
2720            }
2721            // The re-ask only makes sense as a resume: `resume_after_drop`
2722            // says nothing about the task, trusting the seat to still hold it.
2723            // Without a session to resume — sessions disabled, or this CLI's
2724            // drop shape happened not to carry a session id — that prompt
2725            // would open a brand-new conversation with no context at all,
2726            // which is worse than leaving this as the ordinary failure it
2727            // already is.
2728            if !has_context(&job.spec, seat, job.sessions) {
2729                self.state.event(
2730                    "implement",
2731                    format!(
2732                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2733                         is no session left to resume",
2734                        seat.key, dropped.output_tokens, dropped.why
2735                    ),
2736                );
2737                continue;
2738            }
2739            self.state.event(
2740                "implement",
2741                format!(
2742                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2743                     conversation",
2744                    seat.key, dropped.output_tokens, dropped.why
2745                ),
2746            );
2747            let mut retry = job.clone();
2748            retry.seat = seat.clone();
2749            retry.prompt = prompt::resume_after_drop(&dropped.why);
2750            retry.timeout = retry_budget(job.timeout, true);
2751            retry.stem = format!("{}-resume", job.stem);
2752            let cache = self.state.config.cache_dir();
2753            let ctx = WaveCtx {
2754                carry_seats: false,
2755                pauses: Some((&self.pause, &self.interrupt)),
2756                run: run_id,
2757                node: "implement",
2758                prompts,
2759                cache: cache.as_deref(),
2760                round: None,
2761            };
2762            let (resumed_seat, resumed) =
2763                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2764            *seat = resumed_seat;
2765            *out = resumed;
2766        }
2767    }
2768
2769    /// Fall an implement seat through to the next untried agent in the
2770    /// implementer roster when it lost to quota — or, since the handover was
2771    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2772    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2773    /// quota path itself being unchanged — instead of leaving the
2774    /// seat's loss final the moment one agent's account runs dry.
2775    ///
2776    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2777    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2778    /// truncates to the single slot rotation picked, so a solo task whose one
2779    /// implementer hits quota mid-run used to have nothing else to try. This
2780    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2781    /// unrotated roster — which is the only place the *other* candidates in
2782    /// the machine's roster still exist once `implementers` has been cut down
2783    /// to size.
2784    ///
2785    /// Walks forward from just past the seat's own original position in the
2786    /// roster, never wrapping back to the front: a later candidate slot (say
2787    /// `beta`, the roster's second entry) must fall through to the *next*
2788    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2789    /// almost certainly a different candidate's own agent already — and once
2790    /// the roster's tail is exhausted there is nothing left to fall through
2791    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2792    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2793    /// let this retry that id forever. The loop keeps falling through until
2794    /// an attempt lands something other than `Quota` or the roster's tail
2795    /// runs out of untried ids, at which point the seat is left exactly as
2796    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2797    /// `QuotaLoss` recorded, the candidate failed/empty.
2798    ///
2799    /// `sent` is taken mutably and updated with the fallback agent's spec:
2800    /// `resume_unconfirmed_commands`, which runs after this and also reads
2801    /// `sent`, must see whichever agent actually ended up answering the seat
2802    /// — reading the stale, original spec there would check session
2803    /// eligibility against the wrong CLI and could hand a fallback agent's
2804    /// session id to the agent that just lost the seat to quota.
2805    ///
2806    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2807    /// — `self.seat` only reuses state when the agent id is unchanged, so
2808    /// handing it a different id already gets this for free. Reusing the old
2809    /// seat would resume a different CLI's session as if it were a
2810    /// continuation of this one.
2811    ///
2812    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2813    /// a quota loss cuts an agent off mid-turn, so anything already in the
2814    /// tree is unfinished work, not a completed candidate a re-ask would pay
2815    /// for twice. A dirty tree is rescued into a commit first (the same
2816    /// neutral-identity rescue `implement`'s own outcome loop gives every
2817    /// candidate) so the next agent starts clean.
2818    ///
2819    /// The new agent gets the implementer's full prompt and full
2820    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2821    /// it has no session and no context, and is implementing the task from
2822    /// nothing, unlike a resumed drop which is only restating work already
2823    /// done.
2824    ///
2825    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2826    /// `implement` event, never into `self.state.quota` — that is what
2827    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2828    /// go unspent, and a seat that ultimately recovered on its second or
2829    /// third agent is not the stalled panel that check exists to catch. Only
2830    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2831    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2832    /// outcome loop already has — this helper never pushes to it itself.
2833    async fn resume_seat_handovers(
2834        &mut self,
2835        results: &mut [(usize, SeatState, AgentOutcome)],
2836        sent: &mut [SeatJob],
2837        prompts: &Prompts,
2838        run_id: &str,
2839    ) -> Result<()> {
2840        let instruction = seeded_instruction(&self.state);
2841        let language = self.state.config.graph.language.clone();
2842        let github_language = self.state.config.graph.github_language.clone();
2843        let brief = self
2844            .state
2845            .advice
2846            .as_ref()
2847            .and_then(|a| a.synthesis.as_deref())
2848            .map(str::to_owned);
2849        let attachments = self.state.attachments.clone();
2850        for (wi, seat, out) in results.iter_mut() {
2851            // Who holds the other candidate seats of this wave right now
2852            // (earlier handovers already written back to `sent`).
2853            let others: BTreeSet<String> = sent
2854                .iter()
2855                .enumerate()
2856                .filter(|(j, _)| j != wi)
2857                .map(|(_, j)| j.spec.id.clone())
2858                .collect();
2859            let Some(job) = sent.get_mut(*wi) else {
2860                continue;
2861            };
2862            // Where the seat's own original agent sits in the roster — the
2863            // fallback walk starts just past here, never at the front, so a
2864            // later candidate slot's quota loss does not fall back onto an
2865            // earlier slot's own agent.
2866            let start = self
2867                .roles
2868                .implementer_roster
2869                .iter()
2870                .position(|s| s.id == job.spec.id)
2871                .unwrap_or(0);
2872            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2873            let mut fallback_attempt = 0usize;
2874            let mut prev: Option<FailClass> = None;
2875            while let Some(cur) = FailClass::of(&*out) {
2876                if !should_hand_over(prev.as_ref(), &cur) {
2877                    break;
2878                }
2879                let Some(next) =
2880                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2881                        .cloned()
2882                else {
2883                    break;
2884                };
2885                tried.insert(next.id.clone());
2886                fallback_attempt += 1;
2887
2888                let coordination = self.coordinate(&job.cwd).await?;
2889                if let Ok(r) = git::rescue_commit(
2890                    &job.cwd,
2891                    &format!(
2892                        "magi: candidate {} (uncommitted work before {} fallback)",
2893                        seat.key,
2894                        if cur == FailClass::Quota {
2895                            "quota"
2896                        } else {
2897                            "handover"
2898                        }
2899                    ),
2900                )
2901                .await
2902                {
2903                    self.state.note_withheld("implement", &r.withheld);
2904                }
2905                drop(coordination);
2906
2907                record_handover(
2908                    &mut self.state,
2909                    "implement",
2910                    &seat.key,
2911                    &seat.agent,
2912                    &next.id,
2913                    &cur,
2914                    &fail_reason(&*out),
2915                );
2916                prev = Some(cur.clone());
2917
2918                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2919                self.state.seats.insert(seat.key.clone(), new_seat.clone());
2920                // Kept in sync on `sent` itself, not just the local retry: a
2921                // later helper (`resume_unconfirmed_commands`) reads `sent`
2922                // after this one returns and must see whichever agent is now
2923                // occupying the seat, not the one that just quota'd out —
2924                // otherwise it would judge session/continuation eligibility
2925                // by the wrong CLI and could resend a fallback's session id
2926                // to the agent that lost it the seat in the first place.
2927                job.spec = next.clone();
2928                let mut retry = job.clone();
2929                retry.seat = new_seat;
2930                retry.prompt = prompt::implement_in(
2931                    &instruction,
2932                    &job.cwd.to_string_lossy(),
2933                    &language,
2934                    &github_language,
2935                    brief.as_deref(),
2936                    &attachments,
2937                );
2938                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2939                let cache = self.state.config.cache_dir();
2940                let ctx = WaveCtx {
2941                    carry_seats: false,
2942                    pauses: Some((&self.pause, &self.interrupt)),
2943                    run: run_id,
2944                    node: "implement",
2945                    prompts,
2946                    cache: cache.as_deref(),
2947                    round: None,
2948                };
2949                let (fallback_seat, fallback_out) = run_one(
2950                    retry,
2951                    Arc::clone(&self.sem),
2952                    &ctx,
2953                    &mut self.state,
2954                    fallback_attempt,
2955                )
2956                .await;
2957                *seat = fallback_seat;
2958                *out = fallback_out;
2959            }
2960        }
2961        Ok(())
2962    }
2963
2964    /// Ask an implement seat's own CLI to confirm what it started, once, when
2965    /// its reply reported a command whose completion status it never
2966    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
2967    /// that does and does not mean.
2968    ///
2969    /// The completion contract this task asks for, extended to `implement`
2970    /// with the same signal `continue_fix_report` reads for the fixer,
2971    /// rather than a keyword search over the reply or a hard requirement on
2972    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
2973    /// a candidate's CLI turn ended cleanly while a test run it had started
2974    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
2975    /// named in it at all is untouched by this: `commands` is empty, so
2976    /// there is nothing to be unconfirmed.
2977    ///
2978    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
2979    /// this is not about recovering edits that might already be on disk, it
2980    /// is about a result the seat itself never vouched for, which resuming
2981    /// asks for regardless of what the tree already holds. Bounded to one
2982    /// attempt for the same reason `resume_undelivered` is — this is the
2983    /// most expensive node in the graph — and a seat that still cannot
2984    /// confirm on that attempt is left as whatever its (possibly still
2985    /// unconfirmed) reply says; this does not invent a new "failed" reason
2986    /// for a candidate that otherwise produced a real, committed change.
2987    async fn resume_unconfirmed_commands(
2988        &mut self,
2989        results: &mut [(usize, SeatState, AgentOutcome)],
2990        sent: &[SeatJob],
2991        prompts: &Prompts,
2992        run_id: &str,
2993    ) {
2994        for (wi, seat, out) in results.iter_mut() {
2995            let AgentOutcome::Ok(o) = &*out else {
2996                continue;
2997            };
2998            if !has_unconfirmed_command(&o.commands) {
2999                continue;
3000            }
3001            let Some(job) = sent.get(*wi) else { continue };
3002            if !has_context(&job.spec, seat, job.sessions) {
3003                self.state.event(
3004                    "implement",
3005                    format!(
3006                        "{}: the reply named a command whose own CLI never confirmed the exit \
3007                         status of, but there is no session left to resume",
3008                        seat.key
3009                    ),
3010                );
3011                continue;
3012            }
3013            self.state.event(
3014                "implement",
3015                format!(
3016                    "{}: the reply named a command whose own CLI never confirmed the exit \
3017                     status of; resuming the conversation",
3018                    seat.key
3019                ),
3020            );
3021            let mut retry = job.clone();
3022            retry.seat = seat.clone();
3023            retry.prompt = prompt::resume_incomplete(
3024                "a command in your last reply had no confirmed exit status",
3025            );
3026            retry.timeout = retry_budget(job.timeout, true);
3027            retry.stem = format!("{}-confirm", job.stem);
3028            let cache = self.state.config.cache_dir();
3029            let ctx = WaveCtx {
3030                carry_seats: false,
3031                pauses: Some((&self.pause, &self.interrupt)),
3032                run: run_id,
3033                node: "implement",
3034                prompts,
3035                cache: cache.as_deref(),
3036                round: None,
3037            };
3038            let (resumed_seat, resumed) =
3039                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3040            *seat = resumed_seat;
3041            *out = resumed;
3042        }
3043    }
3044
3045    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3046    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3047    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3048    /// that motivated this.
3049    ///
3050    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3051    /// own nudge loop already covers for judge/review/vote seats, and not a
3052    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3053    /// implement seats: here the CLI turn genuinely finished while the node's
3054    /// own work — the fixer's account of what it did — had not. Gated purely
3055    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3056    /// reply, never on any wording in it, so a fixer whose valid, first-try
3057    /// `FixReport` happens to mention having waited on a background test is
3058    /// never resumed — the `Ok(report)` branch at the call site returns
3059    /// before this is ever invoked.
3060    ///
3061    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3062    /// attempt ([`retry_budget`]), nothing attempted once the session is
3063    /// gone, and a quota hit ends the loop immediately rather than retrying a
3064    /// rate limit that fails the same way again.
3065    async fn continue_fix_report(
3066        &mut self,
3067        mut seat: SeatState,
3068        parse_err: String,
3069        job: &SeatJob,
3070        prompts: &Prompts,
3071        run_id: &str,
3072        round: usize,
3073    ) -> (
3074        SeatState,
3075        Option<FixReport>,
3076        Option<String>,
3077        ContinuationRecord,
3078    ) {
3079        let mut last_err = parse_err;
3080        let mut cumulative_wait_ms = 0u64;
3081        let mut attempts = 0usize;
3082        loop {
3083            if !has_context(&job.spec, &seat, job.sessions) {
3084                self.state.event(
3085                    "fix",
3086                    format!(
3087                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3088                         session left to resume into"
3089                    ),
3090                );
3091                let outcome = if attempts == 0 {
3092                    ContinuationOutcome::NoSession
3093                } else {
3094                    ContinuationOutcome::Exhausted
3095                };
3096                return (
3097                    seat,
3098                    None,
3099                    Some(format!("unparsable fix report: {last_err}")),
3100                    ContinuationRecord {
3101                        attempts,
3102                        cumulative_wait_ms,
3103                        outcome,
3104                    },
3105                );
3106            }
3107            if attempts >= MAX_FIX_CONTINUATIONS {
3108                self.state.event(
3109                    "fix",
3110                    format!(
3111                        "round {round}: fixer's reply still had no adoption report after \
3112                         {attempts} continuation(s) ({last_err}); giving up"
3113                    ),
3114                );
3115                return (
3116                    seat,
3117                    None,
3118                    Some(format!(
3119                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3120                    )),
3121                    ContinuationRecord {
3122                        attempts,
3123                        cumulative_wait_ms,
3124                        outcome: ContinuationOutcome::Exhausted,
3125                    },
3126                );
3127            }
3128            attempts += 1;
3129            self.state.event(
3130                "fix",
3131                format!(
3132                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3133                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3134                ),
3135            );
3136            let mut retry = job.clone();
3137            retry.seat = seat.clone();
3138            retry.prompt = prompt::resume_incomplete(&last_err);
3139            retry.timeout = retry_budget(job.timeout, true);
3140            retry.stem = format!("{}-continue{attempts}", job.stem);
3141            let cache = self.state.config.cache_dir();
3142            let ctx = WaveCtx {
3143                carry_seats: false,
3144                pauses: Some((&self.pause, &self.interrupt)),
3145                run: run_id,
3146                node: "fix",
3147                prompts,
3148                cache: cache.as_deref(),
3149                round: Some(round),
3150            };
3151            let (resumed_seat, resumed_out) = run_one(
3152                retry,
3153                Arc::clone(&self.sem),
3154                &ctx,
3155                &mut self.state,
3156                attempts,
3157            )
3158            .await;
3159            seat = resumed_seat;
3160            match resumed_out {
3161                AgentOutcome::Ok(o) => {
3162                    cumulative_wait_ms += o.duration_ms;
3163                    match verdict::extract_json::<FixReport>(&o.text) {
3164                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3165                            self.state.event(
3166                                "fix",
3167                                format!(
3168                                    "round {round}: fixer's adoption report recovered after \
3169                                     {attempts} continuation(s)"
3170                                ),
3171                            );
3172                            return (
3173                                seat,
3174                                Some(report),
3175                                None,
3176                                ContinuationRecord {
3177                                    attempts,
3178                                    cumulative_wait_ms,
3179                                    outcome: ContinuationOutcome::Resumed,
3180                                },
3181                            );
3182                        }
3183                        // The report parsed, but this same reply's own
3184                        // CommandEvidence — the identical record `state.jobs`
3185                        // renders — names a command whose CLI never
3186                        // confirmed an exit status. Read together, that is
3187                        // not a resolved answer: keep nudging rather than
3188                        // accept a report standing next to a command the
3189                        // seat's own CLI cannot vouch for.
3190                        Ok(_) => {
3191                            last_err = "the reply parsed, but it reported a command whose own CLI \
3192                                 never confirmed an exit status"
3193                                .to_owned();
3194                        }
3195                        Err(e) => last_err = e.to_string(),
3196                    }
3197                }
3198                AgentOutcome::Quota(o) => {
3199                    cumulative_wait_ms += o.duration_ms;
3200                    self.state.quota.push(QuotaLoss {
3201                        seat: seat.key.clone(),
3202                        node: "fix".to_owned(),
3203                        at: Timestamp::now(),
3204                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3205                    });
3206                    self.state.event(
3207                        "fix",
3208                        format!(
3209                            "round {round}: continuation rate limited (quota); not retrying now"
3210                        ),
3211                    );
3212                    return (
3213                        seat,
3214                        None,
3215                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3216                        ContinuationRecord {
3217                            attempts,
3218                            cumulative_wait_ms,
3219                            outcome: ContinuationOutcome::QuotaLost,
3220                        },
3221                    );
3222                }
3223                AgentOutcome::Dropped(o) => {
3224                    cumulative_wait_ms += o.duration_ms;
3225                    let why = o
3226                        .dropped
3227                        .as_ref()
3228                        .map(|d| d.why.as_str())
3229                        .unwrap_or("the CLI ended the stream without delivering its answer");
3230                    last_err = format!("the CLI dropped the stream ({why})");
3231                }
3232                AgentOutcome::Failed(e) => last_err = e,
3233                // The continuation was never started; the report stays lost
3234                // and the caller records that, without any handover.
3235                AgentOutcome::ResourceBlocked(e) => {
3236                    return (
3237                        seat,
3238                        None,
3239                        Some(format!("the continuation was not started: {e}")),
3240                        ContinuationRecord {
3241                            attempts: attempts.saturating_sub(1),
3242                            cumulative_wait_ms,
3243                            outcome: ContinuationOutcome::NoSession,
3244                        },
3245                    );
3246                }
3247            }
3248        }
3249    }
3250
3251    fn after_implement(&mut self) -> Result<()> {
3252        if !self.state.candidate_preservation.is_empty() {
3253            bail!("candidate preservation incomplete; judging is unsafe");
3254        }
3255        // Scan every candidate patch once the set is complete.
3256        if self.state.leaks.is_empty() {
3257            let cfg = self.state.config.blind.clone();
3258            let mut leaks = Vec::new();
3259            for c in &self.state.candidates {
3260                let Some(patch) =
3261                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3262                else {
3263                    continue;
3264                };
3265                leaks.extend(blind::scan(
3266                    &format!("candidate {} patch", c.label),
3267                    &patch,
3268                    &cfg.vendor_tokens,
3269                ));
3270            }
3271            if !leaks.is_empty() {
3272                let summary = leaks
3273                    .iter()
3274                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3275                    .collect::<Vec<_>>()
3276                    .join(", ");
3277                match cfg.on_leak {
3278                    LeakPolicy::Fail => {
3279                        self.state.status = RunStatus::Failed;
3280                        self.state
3281                            .event("blind", format!("vendor text in a patch: {summary}"));
3282                        self.state.leaks = leaks;
3283                        self.state.save()?;
3284                        self.settle_questions();
3285                        bail!(
3286                            "blind.on_leak = \"fail\" and vendor text reached a \
3287                             judged patch: {summary}"
3288                        );
3289                    }
3290                    LeakPolicy::Redact => self.state.event(
3291                        "blind",
3292                        format!("redacting vendor text for judging: {summary}"),
3293                    ),
3294                    LeakPolicy::Warn => self.state.event(
3295                        "blind",
3296                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3297                    ),
3298                }
3299                self.state.leaks = leaks;
3300            }
3301        }
3302
3303        if self.state.viable().is_empty() {
3304            if self.state.all_candidates_verified_noop() {
3305                // Every candidate agreed, with evidence the adoption guard
3306                // accepted, that nothing belongs in this worktree. That is
3307                // not the same fact as a candidate that simply failed to
3308                // write anything, and settling it as an ordinary `Failed`
3309                // (see `SCHEMA`'s doc for schema 10) is what let two of
3310                // task 391f's attempts burn a retry each re-discovering the
3311                // same already-landed fix. Terminal either way, so `judge`
3312                // must never run over an empty candidate set — unlike the
3313                // `Failed` branch below this returns `Ok`, not an error:
3314                // nothing here failed.
3315                self.state.status = RunStatus::VerifiedNoop;
3316                self.state.save()?;
3317                self.settle_questions();
3318                return Ok(());
3319            }
3320            self.state.status = RunStatus::Failed;
3321            self.state.save()?;
3322            self.settle_questions();
3323            bail!("no candidate produced a change; nothing to judge");
3324        }
3325        self.state.status = RunStatus::Judging;
3326        self.state.save()?;
3327        Ok(())
3328    }
3329
3330    // --------------------------------------------------------------- judge
3331
3332    async fn judge(&mut self) -> Result<()> {
3333        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3334        // agent files with `magi task add` name the run that paid for it. The
3335        // prompt overlay is cloned alongside it because the waves borrow it
3336        // while `self` is mutably borrowed by the node's own bookkeeping.
3337        let run_id = self.state.id.clone();
3338        let prompts = self.state.config.prompts.clone();
3339        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3340            return Ok(());
3341        }
3342        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3343        if viable.len() == 1 {
3344            // Recorded so this is a one-time event: `judgements` stays empty
3345            // either way, which without this flag is indistinguishable from
3346            // "not yet judged" on the next reentry — and status is left
3347            // untouched, so a later node's conclusion (e.g. `Blocked` after
3348            // the review budget ran out) survives a resume instead of being
3349            // clobbered back to `Judging` by this node running again.
3350            self.state.judge_skipped = true;
3351            self.state.event(
3352                "judge",
3353                format!(
3354                    "only candidate {} produced a change; judging skipped",
3355                    viable[0].label
3356                ),
3357            );
3358            self.state.save()?;
3359            return Ok(());
3360        }
3361        self.state.status = RunStatus::Judging;
3362
3363        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3364        let language = self.state.config.graph.language.clone();
3365        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3366        let sessions = self.state.config.graph.sessions;
3367        let artifacts = agent::artifacts_dir(&self.state.dir());
3368        let root = self.state.worktree_root();
3369        let base_short = short(&self.state.base_commit);
3370
3371        let mut jobs = Vec::new();
3372        let mut orders = Vec::new();
3373        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3374            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3375            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3376            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3377            let seat_key = format!("judge-{}", j + 1);
3378            let seat = self.seat(&seat_key, &spec.id);
3379            jobs.push(SeatJob {
3380                prompt: prompt::judge(
3381                    &self.state.instruction,
3382                    &views,
3383                    self.roles.judges.len(),
3384                    &base_short,
3385                    &language,
3386                ),
3387                spec,
3388                seat,
3389                cwd: root.join(format!("judge-{}", j + 1)),
3390                timeout,
3391                allow_write: false,
3392                sessions,
3393                artifacts: artifacts.clone(),
3394                stem: format!("judge-{}", j + 1),
3395                handover: None,
3396            });
3397        }
3398
3399        self.state.event(
3400            "judge",
3401            format!(
3402                "{} judges ranking {} candidates blind",
3403                jobs.len(),
3404                viable.len()
3405            ),
3406        );
3407        let labels_for_check = labels.clone();
3408        let mut quota_losses = Vec::new();
3409        let cache = self.state.config.cache_dir();
3410        let ctx = WaveCtx {
3411            carry_seats: false,
3412            pauses: Some((&self.pause, &self.interrupt)),
3413            run: &run_id,
3414            node: "judge",
3415            prompts: &prompts,
3416            cache: cache.as_deref(),
3417            round: None,
3418        };
3419        let results = ask_json_wave::<Ranking>(
3420            jobs,
3421            Arc::clone(&self.sem),
3422            self.state.config.graph.retries,
3423            &self.roles.judge_roster,
3424            &ctx,
3425            &mut quota_losses,
3426            &mut self.state,
3427            &move |r: &Ranking| r.validate(&labels_for_check),
3428        )
3429        .await;
3430        if self.state.resource_block.is_some() {
3431            return self.stop_for_resource();
3432        }
3433        self.state.quota.extend(quota_losses);
3434
3435        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3436            let agent_id = seat.agent.clone();
3437            self.state.seats.insert(seat.key.clone(), seat);
3438            let mut record = Judgement {
3439                judge: j + 1,
3440                seat: format!("judge-{}", j + 1),
3441                agent: agent_id,
3442                ranking: Vec::new(),
3443                reasons: BTreeMap::new(),
3444                confidence: None,
3445                strengths: BTreeMap::new(),
3446                order: orders[j].clone(),
3447                failed: None,
3448                duration_ms: 0,
3449            };
3450            match res {
3451                Ok((ranking, out)) => {
3452                    record.ranking = ranking.normalized();
3453                    record.reasons = ranking.reasons;
3454                    record.strengths = ranking.strengths;
3455                    record.confidence = ranking.confidence;
3456                    record.duration_ms = out.duration_ms;
3457                    self.state.event(
3458                        "judge",
3459                        format!(
3460                            "judge {} ranked {}",
3461                            j + 1,
3462                            record.ranking.iter().collect::<String>()
3463                        ),
3464                    );
3465                }
3466                Err(e) => {
3467                    record.failed = Some(e.to_string());
3468                    self.state
3469                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3470                }
3471            }
3472            self.state.judgements.push(record);
3473            self.state.save()?;
3474        }
3475        Ok(())
3476    }
3477
3478    // ---------------------------------------------------------- deliberate
3479
3480    async fn deliberate(&mut self) -> Result<()> {
3481        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3482        // agent files with `magi task add` name the run that paid for it. The
3483        // prompt overlay is cloned alongside it because the waves borrow it
3484        // while `self` is mutably borrowed by the node's own bookkeeping.
3485        let run_id = self.state.id.clone();
3486        let prompts = self.state.config.prompts.clone();
3487        if !self.state.deliberation.is_empty() {
3488            return Ok(());
3489        }
3490        let tops: Vec<char> = self
3491            .state
3492            .judgements
3493            .iter()
3494            .filter_map(|j| j.ranking.first().copied())
3495            .collect();
3496        let rounds = self.state.config.graph.deliberate_rounds;
3497        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3498            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3499                self.state.event(
3500                    "deliberate",
3501                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3502                );
3503            }
3504            self.state.status = RunStatus::Voting;
3505            self.state.save()?;
3506            return Ok(());
3507        }
3508
3509        self.state.status = RunStatus::Deliberating;
3510        self.state.event(
3511            "deliberate",
3512            format!(
3513                "split: first choices were {} — opening {rounds} round(s)",
3514                tops.iter().collect::<String>()
3515            ),
3516        );
3517
3518        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3519        let language = self.state.config.graph.language.clone();
3520        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3521        let sessions = self.state.config.graph.sessions;
3522        let artifacts = agent::artifacts_dir(&self.state.dir());
3523        let root = self.state.worktree_root();
3524        let base_short = short(&self.state.base_commit);
3525
3526        // Judges argue in sequence so that a turn can answer the one before it;
3527        // that is the difference between deliberation and three parallel
3528        // monologues.
3529        for round in 1..=rounds {
3530            let mut turns: Vec<DeliberationTurn> = Vec::new();
3531            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3532                if self.state.judgements[j].failed.is_some() {
3533                    continue;
3534                }
3535                let seat_key = format!("judge-{}", j + 1);
3536                let spec = self.occupant(&seat_key, spec);
3537                let mut seat = self.seat(&seat_key, &spec.id);
3538                let transcript = self.transcript(&turns, j);
3539                let build = |context: Option<&str>| {
3540                    prompt::deliberate(
3541                        &self.state.instruction,
3542                        context,
3543                        &transcript,
3544                        round,
3545                        rounds,
3546                        &language,
3547                    )
3548                };
3549                let block = self.candidate_block(&viable, &base_short);
3550                let full = build(Some(&block));
3551                let text = if has_context(&spec, &seat, sessions) {
3552                    build(None)
3553                } else {
3554                    full.clone()
3555                };
3556                let job = SeatJob {
3557                    spec,
3558                    seat: seat.clone(),
3559                    prompt: text,
3560                    cwd: root.join(format!("judge-{}", j + 1)),
3561                    timeout,
3562                    allow_write: false,
3563                    sessions,
3564                    artifacts: artifacts.clone(),
3565                    stem: format!("delib-{round}-judge-{}", j + 1),
3566                    handover: Some(full),
3567                };
3568                let cache = self.state.config.cache_dir();
3569                let ctx = WaveCtx {
3570                    carry_seats: false,
3571                    pauses: Some((&self.pause, &self.interrupt)),
3572                    run: &run_id,
3573                    node: "deliberate",
3574                    prompts: &prompts,
3575                    cache: cache.as_deref(),
3576                    round: None,
3577                };
3578                // A turn is never nudged (`retries` 0): a failed seat is
3579                // handed to the next roster agent, which gets the full
3580                // context. An empty answer is a turn, not a failure.
3581                let mut losses = Vec::new();
3582                let mut results = ask_wave_with::<String>(
3583                    vec![job],
3584                    Arc::clone(&self.sem),
3585                    0,
3586                    &self.roles.judge_roster,
3587                    &ctx,
3588                    &mut losses,
3589                    &mut self.state,
3590                    &|text: &str| {
3591                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3592                    },
3593                )
3594                .await;
3595                self.state.quota.extend(losses);
3596                let (updated, res, _) = results.pop().expect("one job in, one result out");
3597                seat = updated;
3598                let agent_id = seat.agent.clone();
3599                self.state.seats.insert(seat.key.clone(), seat);
3600                let body = match res {
3601                    Ok((body, _)) => body,
3602                    // Skip the seat; a CLI's raw error JSON is never read as
3603                    // this judge's position.
3604                    Err(e) => {
3605                        self.state
3606                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3607                        continue;
3608                    }
3609                };
3610                let tentative = verdict::extract_json::<Position>(&body)
3611                    .ok()
3612                    .and_then(|p| p.tentative)
3613                    .and_then(|s| s.trim().chars().next())
3614                    .map(|c| c.to_ascii_uppercase());
3615                self.state.event(
3616                    "deliberate",
3617                    format!(
3618                        "round {round}: judge {} now favours {}",
3619                        j + 1,
3620                        tentative.map_or("—".to_owned(), |c| c.to_string())
3621                    ),
3622                );
3623                turns.push(DeliberationTurn {
3624                    judge: j + 1,
3625                    agent: agent_id,
3626                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3627                    tentative,
3628                });
3629            }
3630            self.state
3631                .deliberation
3632                .push(DeliberationRound { round, turns });
3633            self.state.save()?;
3634        }
3635
3636        self.state.status = RunStatus::Voting;
3637        self.state.save()?;
3638        Ok(())
3639    }
3640
3641    // ---------------------------------------------------------------- vote
3642
3643    async fn vote(&mut self) -> Result<()> {
3644        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3645        // agent files with `magi task add` name the run that paid for it. The
3646        // prompt overlay is cloned alongside it because the waves borrow it
3647        // while `self` is mutably borrowed by the node's own bookkeeping.
3648        let run_id = self.state.id.clone();
3649        let prompts = self.state.config.prompts.clone();
3650        if !self.state.votes.is_empty() {
3651            return Ok(());
3652        }
3653        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3654        if viable.len() == 1 {
3655            return Ok(());
3656        }
3657        self.state.status = RunStatus::Voting;
3658
3659        let language = self.state.config.graph.language.clone();
3660        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3661        let sessions = self.state.config.graph.sessions;
3662        let artifacts = agent::artifacts_dir(&self.state.dir());
3663        let root = self.state.worktree_root();
3664        let base_short = short(&self.state.base_commit);
3665        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3666
3667        let mut jobs = Vec::new();
3668        let mut seats_at = Vec::new();
3669        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3670            if self
3671                .state
3672                .judgements
3673                .get(j)
3674                .is_some_and(|r| r.failed.is_some())
3675            {
3676                continue;
3677            }
3678            let seat_key = format!("judge-{}", j + 1);
3679            let spec = self.occupant(&seat_key, spec);
3680            let seat = self.seat(&seat_key, &spec.id);
3681            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3682            let text = if has_context(&spec, &seat, sessions) {
3683                prompt::final_vote(&viable, &language)
3684            } else {
3685                full.clone()
3686            };
3687            jobs.push(SeatJob {
3688                spec,
3689                seat,
3690                prompt: text,
3691                cwd: root.join(format!("judge-{}", j + 1)),
3692                timeout,
3693                allow_write: false,
3694                sessions,
3695                artifacts: artifacts.clone(),
3696                stem: format!("vote-judge-{}", j + 1),
3697                handover: Some(full),
3698            });
3699            seats_at.push(j);
3700        }
3701
3702        self.state.event(
3703            "vote",
3704            format!(
3705                "collecting {} final votes one by one, privately",
3706                jobs.len()
3707            ),
3708        );
3709        let allowed = viable.clone();
3710        let mut quota_losses = Vec::new();
3711        let cache = self.state.config.cache_dir();
3712        let ctx = WaveCtx {
3713            carry_seats: false,
3714            pauses: Some((&self.pause, &self.interrupt)),
3715            run: &run_id,
3716            node: "vote",
3717            prompts: &prompts,
3718            cache: cache.as_deref(),
3719            round: None,
3720        };
3721        let results = ask_json_wave::<FinalVote>(
3722            jobs,
3723            Arc::clone(&self.sem),
3724            self.state.config.graph.retries,
3725            &self.roles.judge_roster,
3726            &ctx,
3727            &mut quota_losses,
3728            &mut self.state,
3729            &move |v: &FinalVote| match v.label() {
3730                Some(c) if allowed.contains(&c) => Ok(()),
3731                other => bail!("vote {other:?} is not one of {allowed:?}"),
3732            },
3733        )
3734        .await;
3735        if self.state.resource_block.is_some() {
3736            return self.stop_for_resource();
3737        }
3738        self.state.quota.extend(quota_losses);
3739
3740        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3741            let agent_id = seat.agent.clone();
3742            self.state.seats.insert(seat.key.clone(), seat);
3743            let initial = self
3744                .state
3745                .judgements
3746                .get(j)
3747                .and_then(|r| r.ranking.first().copied());
3748            let mut record = VoteRecord {
3749                judge: j + 1,
3750                agent: agent_id,
3751                vote: None,
3752                reason: String::new(),
3753                changed: false,
3754            };
3755            match res {
3756                Ok((v, _)) => {
3757                    record.vote = v.label();
3758                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3759                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3760                    self.state.event(
3761                        "vote",
3762                        format!(
3763                            "judge {} voted {}{}",
3764                            j + 1,
3765                            record.vote.unwrap_or('?'),
3766                            if record.changed { " (changed)" } else { "" }
3767                        ),
3768                    );
3769                }
3770                Err(e) => {
3771                    self.state
3772                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3773                }
3774            }
3775            self.state.votes.push(record);
3776            self.state.save()?;
3777        }
3778        Ok(())
3779    }
3780
3781    // --------------------------------------------------------------- tally
3782
3783    fn tally(&mut self) -> Result<()> {
3784        if self.state.tally.is_some() {
3785            return Ok(());
3786        }
3787        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3788        let tops: Vec<char> = self
3789            .state
3790            .judgements
3791            .iter()
3792            .filter_map(|j| j.ranking.first().copied())
3793            .collect();
3794        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3795
3796        // A judge whose private vote failed still counted once, in the initial
3797        // ranking; using it beats discarding a whole seat.
3798        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3799        let mut cast: Vec<char> = Vec::new();
3800        for (i, j) in self.state.judgements.iter().enumerate() {
3801            let vote = self
3802                .state
3803                .votes
3804                .iter()
3805                .find(|v| v.judge == i + 1)
3806                .and_then(|v| v.vote)
3807                .or_else(|| j.ranking.first().copied());
3808            if let Some(v) = vote {
3809                *first_choice.entry(v).or_insert(0) += 1;
3810                cast.push(v);
3811            }
3812        }
3813
3814        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3815        for j in &self.state.judgements {
3816            let n = j.ranking.len();
3817            for (pos, label) in j.ranking.iter().enumerate() {
3818                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3819            }
3820        }
3821
3822        let best = first_choice.values().copied().max().unwrap_or(0);
3823        let mut leaders: Vec<char> = first_choice
3824            .iter()
3825            .filter(|(_, v)| **v == best)
3826            .map(|(k, _)| *k)
3827            .collect();
3828        let mut tie_break = None;
3829        if leaders.len() > 1 {
3830            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3831            let borda_leaders: Vec<char> = leaders
3832                .iter()
3833                .copied()
3834                .filter(|l| borda[l] == top_borda)
3835                .collect();
3836            tie_break = Some(if borda_leaders.len() == 1 {
3837                format!(
3838                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3839                    leaders.len()
3840                )
3841            } else {
3842                format!(
3843                    "{} way tie on both first-choice votes and Borda points, broken by label order",
3844                    leaders.len()
3845                )
3846            });
3847            leaders = borda_leaders;
3848            leaders.sort_unstable();
3849        }
3850        let winner = *leaders
3851            .first()
3852            .or(viable.first())
3853            .context("no candidate to declare a winner from")?;
3854
3855        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3856        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3857        let deliberated = !self.state.deliberation.is_empty();
3858
3859        // Whose verdict is this? A rate-limited seat is absent even if it
3860        // ranked before the limit hit, so presence is measured against the
3861        // recorded losses, not just "did a ranking ever appear".
3862        let quota_seats: std::collections::BTreeSet<&str> =
3863            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3864        let mut present = 0usize;
3865        for (i, j) in self.state.judgements.iter().enumerate() {
3866            if quota_seats.contains(j.seat.as_str()) {
3867                continue;
3868            }
3869            let ranked = !j.ranking.is_empty() && j.failed.is_none();
3870            let voted = self
3871                .state
3872                .votes
3873                .iter()
3874                .any(|v| v.judge == i + 1 && v.vote.is_some());
3875            if ranked || voted {
3876                present += 1;
3877            }
3878        }
3879        // Strict majority of the configured panel. A bare majority is real
3880        // signal we can act on, while a minority verdict must never stand in
3881        // for a healthy one. A one-candidate run needs no panel at all, and
3882        // `judges` stays `0` rather than the roster size a panel that never
3883        // sat would otherwise be credited with.
3884        let needs_quorum = viable.len() > 1;
3885        let judges_total = if needs_quorum {
3886            self.roles.judges.len()
3887        } else {
3888            0
3889        };
3890        let quorum = if needs_quorum {
3891            judges_total / 2 + 1
3892        } else {
3893            0
3894        };
3895        let met_quorum = !needs_quorum || present >= quorum;
3896        let uncontested = (!needs_quorum).then(|| {
3897            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3898        });
3899
3900        self.state.event(
3901            "tally",
3902            match &uncontested {
3903                Some(reason) => format!("winner {winner} — {reason}"),
3904                None => format!(
3905                    "winner {winner} — votes {} | initial {} | {} changed | \
3906                     {present}/{judges_total} judges{}",
3907                    first_choice
3908                        .iter()
3909                        .map(|(k, v)| format!("{k}:{v}"))
3910                        .collect::<Vec<_>>()
3911                        .join(" "),
3912                    if unanimous_initial {
3913                        "unanimous"
3914                    } else {
3915                        "split"
3916                    },
3917                    changed_votes,
3918                    if met_quorum {
3919                        String::new()
3920                    } else {
3921                        format!(" — below quorum ({quorum} required)")
3922                    },
3923                ),
3924            },
3925        );
3926        if !met_quorum {
3927            self.state.event(
3928                "stall",
3929                format!(
3930                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3931                     the run stops here, resumable"
3932                ),
3933            );
3934        }
3935        self.state.tally = Some(Tally {
3936            first_choice,
3937            borda,
3938            winner,
3939            rankings: tops.len(),
3940            unanimous_initial,
3941            deliberated,
3942            changed_votes,
3943            unanimous_final,
3944            tie_break,
3945            judges: judges_total,
3946            present,
3947            quorum,
3948            met_quorum,
3949            uncontested,
3950        });
3951        self.state.status = if met_quorum {
3952            RunStatus::Reviewing
3953        } else {
3954            RunStatus::Stalled
3955        };
3956        self.state.save()?;
3957        Ok(())
3958    }
3959
3960    // ------------------------------------------------------------- recover
3961
3962    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
3963    /// resumed toward completion once the transient cause clears.
3964    ///
3965    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
3966    /// it toward the quorum, which is exactly the set of seats whose absence
3967    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
3968    /// not depend on which node happened to hit the limit), or an ordinary
3969    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
3970    /// seat is never disturbed.
3971    ///
3972    /// A seat that now answers with a usable ranking is "recovered": its
3973    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
3974    /// `tally` counts it present again), and its vote re-collected. A seat that
3975    /// still fails keeps its loss and stays absent.
3976    ///
3977    /// Returns `true` when the re-tally restores the quorum (the run may proceed
3978    /// to review/gate/merge), `false` when it is still below quorum (the run
3979    /// stays `Stalled`, still resumable for a later retry).
3980    #[allow(clippy::too_many_lines)]
3981    async fn recover_stall(&mut self) -> Result<bool> {
3982        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3983        // agent files with `magi task add` name the run that paid for it. The
3984        // prompt overlay is cloned alongside it because the waves borrow it
3985        // while `self` is mutably borrowed by the node's own bookkeeping.
3986        let run_id = self.state.id.clone();
3987        let prompts = self.state.config.prompts.clone();
3988        // Absent seats = quota-lost at any node, or failed outright. Mirroring
3989        // `tally`'s presence test (rather than the old quota-judge/vote filter)
3990        // is what keeps a non-quota collapse — or a quota loss recorded at the
3991        // deliberate node — from being a permanent dead-end on `--resume`.
3992        let quota_seats: BTreeSet<&str> =
3993            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3994        let absent: Vec<String> = self
3995            .state
3996            .judgements
3997            .iter()
3998            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3999            .map(|j| j.seat.clone())
4000            .collect();
4001        if absent.is_empty() {
4002            return Ok(false);
4003        }
4004        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4005        if viable.len() <= 1 {
4006            return Ok(false);
4007        }
4008        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4009        let language = self.state.config.graph.language.clone();
4010        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4011        let sessions = self.state.config.graph.sessions;
4012        let artifacts = agent::artifacts_dir(&self.state.dir());
4013        let root = self.state.worktree_root();
4014        let base_short = short(&self.state.base_commit);
4015        let candidates: Vec<Candidate> = viable.clone();
4016
4017        // Map each absent seat key to its 0-based position in `roles.judges`.
4018        let mut positions: Vec<usize> = absent
4019            .iter()
4020            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4021            .collect();
4022        if positions.is_empty() {
4023            return Ok(false);
4024        }
4025        positions.sort_unstable();
4026        positions.dedup();
4027
4028        // Re-rank the lost seats, one blind prompt each.
4029        let mut judge_jobs = Vec::new();
4030        for &j in &positions {
4031            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4032            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4033            let seat_key = format!("judge-{}", j + 1);
4034            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4035            let seat = self.seat(&seat_key, &spec.id);
4036            judge_jobs.push(SeatJob {
4037                spec,
4038                seat,
4039                prompt: prompt::judge(
4040                    &self.state.instruction,
4041                    &views,
4042                    self.roles.judges.len(),
4043                    &base_short,
4044                    &language,
4045                ),
4046                cwd: root.join(seat_key),
4047                timeout,
4048                allow_write: false,
4049                sessions,
4050                artifacts: artifacts.clone(),
4051                stem: format!("judge-{}-recover", j + 1),
4052                handover: None,
4053            });
4054        }
4055
4056        let labels_for_check = labels.clone();
4057        let mut judge_losses = Vec::new();
4058        let retries = self.state.config.graph.retries;
4059        let cache = self.state.config.cache_dir();
4060        let ctx = WaveCtx {
4061            carry_seats: false,
4062            pauses: Some((&self.pause, &self.interrupt)),
4063            run: &run_id,
4064            node: "judge",
4065            prompts: &prompts,
4066            cache: cache.as_deref(),
4067            round: None,
4068        };
4069        let results = ask_json_wave::<Ranking>(
4070            judge_jobs,
4071            Arc::clone(&self.sem),
4072            retries,
4073            &self.roles.judge_roster,
4074            &ctx,
4075            &mut judge_losses,
4076            &mut self.state,
4077            &move |r: &Ranking| r.validate(&labels_for_check),
4078        )
4079        .await;
4080        if self.state.resource_block.is_some() {
4081            self.stop_for_resource()?;
4082            return Ok(false);
4083        }
4084
4085        // Refresh the judgement of every seat that ranked again.
4086        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4087        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4088            let agent_id = seat.agent.clone();
4089            self.state.seats.insert(seat.key.clone(), seat);
4090            let record = &mut self.state.judgements[j];
4091            match res {
4092                Ok((ranking, out)) => {
4093                    record.agent = agent_id;
4094                    record.ranking = ranking.normalized();
4095                    record.reasons = ranking.reasons;
4096                    record.strengths = ranking.strengths;
4097                    record.confidence = ranking.confidence;
4098                    record.failed = None;
4099                    record.duration_ms = out.duration_ms;
4100                    recovered.insert(j);
4101                    self.state.event(
4102                        "recover",
4103                        format!("judge {} ranked again after the limit", j + 1),
4104                    );
4105                }
4106                Err(e) => {
4107                    self.state
4108                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4109                }
4110            }
4111        }
4112
4113        // Re-ask the votes of the seats that recovered a ranking.
4114        let mut vote_jobs = Vec::new();
4115        let mut vote_pos: Vec<usize> = Vec::new();
4116        for &j in &recovered {
4117            let seat_key = format!("judge-{}", j + 1);
4118            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4119            let seat = self.seat(&seat_key, &spec.id);
4120            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4121            let text = if has_context(&spec, &seat, sessions) {
4122                prompt::final_vote(&labels, &language)
4123            } else {
4124                full.clone()
4125            };
4126            vote_jobs.push(SeatJob {
4127                spec,
4128                seat,
4129                prompt: text,
4130                cwd: root.join(seat_key),
4131                timeout,
4132                allow_write: false,
4133                sessions,
4134                artifacts: artifacts.clone(),
4135                stem: format!("vote-judge-{}-recover", j + 1),
4136                handover: Some(full),
4137            });
4138            vote_pos.push(j);
4139        }
4140        let allowed = labels.clone();
4141        let mut vote_losses = Vec::new();
4142        let vote_retries = self.state.config.graph.retries;
4143        let vote_cache = self.state.config.cache_dir();
4144        let ctx = WaveCtx {
4145            carry_seats: false,
4146            pauses: Some((&self.pause, &self.interrupt)),
4147            run: &run_id,
4148            node: "vote",
4149            prompts: &prompts,
4150            cache: vote_cache.as_deref(),
4151            round: None,
4152        };
4153        let votes = ask_json_wave::<FinalVote>(
4154            vote_jobs,
4155            Arc::clone(&self.sem),
4156            vote_retries,
4157            &self.roles.judge_roster,
4158            &ctx,
4159            &mut vote_losses,
4160            &mut self.state,
4161            &move |v: &FinalVote| match v.label() {
4162                Some(c) if allowed.contains(&c) => Ok(()),
4163                other => bail!("vote {other:?} is not one of {allowed:?}"),
4164            },
4165        )
4166        .await;
4167        if self.state.resource_block.is_some() {
4168            self.stop_for_resource()?;
4169            return Ok(false);
4170        }
4171        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4172            let agent_id = seat.agent.clone();
4173            self.state.seats.insert(seat.key.clone(), seat);
4174            match res {
4175                Ok((v, _)) => {
4176                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4177                        rec.vote = v.label();
4178                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4179                    } else {
4180                        self.state.votes.push(VoteRecord {
4181                            judge: j + 1,
4182                            agent: agent_id,
4183                            vote: v.label(),
4184                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4185                            changed: false,
4186                        });
4187                    }
4188                    self.state.event(
4189                        "recover",
4190                        format!("judge {} voted again after the limit", j + 1),
4191                    );
4192                }
4193                Err(e) => {
4194                    self.state
4195                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4196                }
4197            }
4198        }
4199
4200        // A seat that ranked again is present even if its re-vote failed —
4201        // `tally` falls back to the initial ranking's first choice — so clear
4202        // its quota loss. Seats that still fail keep theirs and stay absent.
4203        let recovered_keys: BTreeSet<String> = recovered
4204            .iter()
4205            .map(|&j| format!("judge-{}", j + 1))
4206            .collect();
4207        self.state
4208            .quota
4209            .retain(|q| !recovered_keys.contains(&q.seat));
4210        // A seat that hit the limit again is a fresh loss, not the old one:
4211        // replace the stale entry so the history stays one-per-seat and the
4212        // daemon can tell this attempt's loss from a previous session's.
4213        for loss in judge_losses.into_iter().chain(vote_losses) {
4214            if recovered_keys.contains(&loss.seat) {
4215                continue;
4216            }
4217            self.state.quota.retain(|q| q.seat != loss.seat);
4218            self.state.quota.push(loss);
4219        }
4220
4221        // Recompute the verdict from the refreshed panel.
4222        self.state.tally = None;
4223        self.tally()?;
4224        Ok(self
4225            .state
4226            .tally
4227            .as_ref()
4228            .map(|t| t.met_quorum)
4229            .unwrap_or(false))
4230    }
4231
4232    // ---------------------------------------------------------------- graft
4233
4234    /// Port the strengths the judges attributed to the losing candidates onto
4235    /// the winner, once, between the tally and the review loop.
4236    ///
4237    /// Best-effort by construction: it returns nothing and fails nothing. A
4238    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4239    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4240    /// the run carries on as if the phase were off. See [`graft`] for the
4241    /// decisions and `AGENTS.md` for the invariants.
4242    ///
4243    /// The one exception is a rollback that could not be confirmed: the
4244    /// rejected commit may still be the winner's tip, so the run is blocked
4245    /// (resumable; the record stays `Running`, which makes a resume retry the
4246    /// restore) instead of reviewing, gating and merging it.
4247    async fn graft(&mut self) -> Result<()> {
4248        let Err(e) = self.graft_inner().await else {
4249            return Ok(());
4250        };
4251        if e.is::<crate::repo_lock::ResourceBlocked>() {
4252            return Err(e);
4253        }
4254        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4255            self.state.status = RunStatus::Blocked;
4256            self.state.event("graft", format!("blocked: {e:#}"));
4257            self.state.save()?;
4258            return Err(e);
4259        }
4260        // An error after the `Running` record was saved may have left the
4261        // fixer's work in the tree: the record must not stay unsettled
4262        // (a later resume would reset past the review fixes), so confirm the
4263        // restore now, or block.
4264        if let Some(rec) = self.state.graft.clone()
4265            && !rec.outcome.settled()
4266        {
4267            if let Some(winner) = self.state.winner().cloned() {
4268                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4269                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4270                        return Err(re);
4271                    }
4272                    self.state.status = RunStatus::Blocked;
4273                    self.state.event("graft", format!("blocked: {re:#}"));
4274                    self.state.save()?;
4275                    return Err(re);
4276                }
4277            }
4278            if let Some(g) = self.state.graft.as_mut() {
4279                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4280            }
4281        }
4282        self.state.event("graft", format!("skipped: {e:#}"));
4283        if let Err(e) = self.state.save() {
4284            tracing::warn!("save after graft: {e:#}");
4285        }
4286        Ok(())
4287    }
4288
4289    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4290    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4291    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4292        let _coordination = self.coordinate(wt).await?;
4293        crate::recovery::restore_repair(wt, pre)
4294            .await
4295            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4296    }
4297
4298    /// A skip is a run event, recorded once however often the node is re-entered.
4299    fn graft_skipped(&mut self, why: &str) {
4300        let message = format!("skipped: {why}");
4301        if !self
4302            .state
4303            .events
4304            .iter()
4305            .any(|e| e.node == "graft" && e.message == message)
4306        {
4307            self.state.event("graft", message);
4308        }
4309    }
4310
4311    async fn graft_inner(&mut self) -> Result<()> {
4312        // A round that was started and never finished (crash, kill): the
4313        // fixer may have left anything in the tree, so put the winner back.
4314        if let Some(rec) = self.state.graft.clone()
4315            && !rec.outcome.settled()
4316        {
4317            // Reviews exist only if the graft ended without its record being
4318            // settled (a swallowed error): the tree has moved on since, and
4319            // a reset to `pre` would discard the review fixes.
4320            if self.state.reviews.is_empty()
4321                && let Some(winner) = self.state.winner().cloned()
4322            {
4323                self.graft_restore(&winner.worktree, &rec.pre).await?;
4324            }
4325            if rec.outcome == GraftOutcome::WaitingForRestore {
4326                self.state.graft = None;
4327                self.state.save()?;
4328            } else {
4329                if let Some(g) = self.state.graft.as_mut() {
4330                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4331                }
4332                self.state.event(
4333                    "graft",
4334                    "rolled back: the previous round was interrupted; the winner is untouched",
4335                );
4336                self.state.save()?;
4337                return Ok(());
4338            }
4339        }
4340        let plan = match graft::plan(&self.state) {
4341            Ok(plan) => plan,
4342            Err(skip) => {
4343                if skip != graft::Skip::AlreadyDone {
4344                    self.graft_skipped(skip.reason());
4345                    self.state.save()?;
4346                }
4347                return Ok(());
4348            }
4349        };
4350        let Some(winner) = self.state.winner().cloned() else {
4351            return Ok(());
4352        };
4353        let wt = winner.worktree.clone();
4354        if !git::is_clean(&wt).await? {
4355            self.graft_skipped("the winner's worktree has uncommitted changes");
4356            self.state.save()?;
4357            return Ok(());
4358        }
4359        let pre = git::rev_parse(&wt, "HEAD").await?;
4360        // Saved before the fixer is called: a stop from here on is repaired
4361        // by the branch above on the next entry.
4362        self.state.graft = Some(GraftRecord {
4363            pre: pre.clone(),
4364            post: None,
4365            lines: None,
4366            limit: None,
4367            agent: None,
4368            outcome: GraftOutcome::Running,
4369            requested: graft::requested(&plan),
4370            adoption: None,
4371            adoption_error: None,
4372        });
4373        self.state.event(
4374            "graft",
4375            format!(
4376                "porting strengths of candidate(s) {} onto {}",
4377                plan.strengths
4378                    .iter()
4379                    .map(|(l, _, _)| l.to_string())
4380                    .collect::<Vec<_>>()
4381                    .join(", "),
4382                plan.winner
4383            ),
4384        );
4385        self.state.save()?;
4386
4387        let fix_prompt = prompt::graft(
4388            &self.state.instruction,
4389            plan.winner,
4390            &plan.strengths,
4391            &self.state.config.graph.language,
4392        );
4393        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4394        let sessions = self.state.config.graph.sessions;
4395        let artifacts = agent::artifacts_dir(&self.state.dir());
4396        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4397        let (_, seat, out) = self
4398            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4399                SeatJob {
4400                    prompt: fix_prompt.clone(),
4401                    spec,
4402                    seat,
4403                    cwd: wt.clone(),
4404                    timeout,
4405                    allow_write: true,
4406                    sessions,
4407                    artifacts: artifacts.clone(),
4408                    stem: "graft".to_owned(),
4409                    handover: None,
4410                }
4411            })
4412            .await;
4413        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4414            // Preserve the need to restore earlier fallback edits without
4415            // confusing an unstarted seat with an interrupted invocation.
4416            self.state.graft.as_mut().expect("graft record").outcome =
4417                GraftOutcome::WaitingForRestore;
4418            return self.stop_for_resource();
4419        }
4420        let mut why: Option<String> = None;
4421        match &out {
4422            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4423                why = Some("the fixer produced nothing usable".to_owned());
4424            }
4425            AgentOutcome::Ok(_) => {}
4426            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4427            AgentOutcome::Quota(o) => {
4428                self.state.quota.push(QuotaLoss {
4429                    seat: seat.key.clone(),
4430                    node: "graft".to_owned(),
4431                    at: Timestamp::now(),
4432                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4433                });
4434                why = Some("rate limited (quota)".to_owned());
4435            }
4436            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4437                why = Some(format!(
4438                    "the fixer failed: {}",
4439                    e.lines()
4440                        .next()
4441                        .unwrap_or("")
4442                        .chars()
4443                        .take(160)
4444                        .collect::<String>()
4445                ));
4446            }
4447        }
4448        let agent_id = seat.agent.clone();
4449        self.state.seats.insert(seat.key.clone(), seat);
4450
4451        let mut post = None;
4452        let mut lines = None;
4453        let _coordination = self.coordinate(&wt).await?;
4454        if why.is_none() {
4455            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4456            if let Ok(r) = &rescued {
4457                self.state.note_withheld("graft", &r.withheld);
4458            }
4459            let head = git::rev_parse(&wt, "HEAD").await?;
4460            // What is measured must be what gets reviewed and gated: a failed
4461            // rescue, or anything still uncommitted (withheld lockfiles
4462            // aside, which are never part of the tree), rejects the graft.
4463            let dirty = match &rescued {
4464                Ok(r) => git::status_porcelain(&wt)
4465                    .await
4466                    .map(|st| {
4467                        st.lines().any(|l| {
4468                            !r.withheld
4469                                .iter()
4470                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4471                        })
4472                    })
4473                    .unwrap_or(true),
4474                Err(_) => true,
4475            };
4476            if dirty {
4477                why = Some("the fixer left changes that could not be committed".to_owned());
4478            } else if head == pre {
4479                why = Some("the fixer changed nothing".to_owned());
4480            } else {
4481                lines = git::changed_lines(&wt, &pre, &head).await;
4482                post = Some(head);
4483            }
4484        }
4485        let (adoption, adoption_error) = match &out {
4486            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4487            _ => (None, None),
4488        };
4489        let requested = graft::requested(&plan);
4490        let outcome = match why {
4491            None => {
4492                self.state.event(
4493                    "graft",
4494                    format!(
4495                        "applied: {}",
4496                        lines.map_or_else(
4497                            || "changed lines unavailable".to_owned(),
4498                            |n| format!("{n} changed lines")
4499                        )
4500                    ),
4501                );
4502                GraftOutcome::Applied
4503            }
4504            Some(why) => {
4505                // Not settled until the restore is confirmed: the record
4506                // stays `Running`, so a resume retries it.
4507                crate::recovery::restore_repair(&wt, &pre)
4508                    .await
4509                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4510                self.state.event(
4511                    "graft",
4512                    format!("rolled back: {why}; the winner is untouched"),
4513                );
4514                GraftOutcome::RolledBack(why)
4515            }
4516        };
4517        self.state.graft = Some(GraftRecord {
4518            pre,
4519            post,
4520            lines,
4521            limit: None,
4522            agent: Some(agent_id),
4523            outcome,
4524            requested,
4525            adoption,
4526            adoption_error,
4527        });
4528        self.state.save()?;
4529        Ok(())
4530    }
4531
4532    // ----------------------------------------------------------------- fold
4533
4534    async fn fold_losers(&mut self) -> Result<()> {
4535        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4536            return Ok(());
4537        };
4538        if !self.state.candidate_preservation.is_empty() {
4539            bail!("candidate preservation incomplete; refusing cleanup");
4540        }
4541        let repo = self.state.repo.clone();
4542        let _coordination = self.coordinate(&repo).await?;
4543        let mut folded = Vec::new();
4544        for i in 0..self.state.candidates.len() {
4545            let c = &self.state.candidates[i];
4546            if c.label == winner || c.folded {
4547                continue;
4548            }
4549            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4550            remove_preserved_worktree(&repo, &wt).await?;
4551            if git::branch_exists(&repo, &branch).await?
4552                && !git::branch_delete(&repo, &branch).await?
4553            {
4554                bail!("candidate branch removal refused");
4555            }
4556            self.state.candidates[i].folded = true;
4557            self.state.save()?;
4558            folded.push(label.to_string());
4559        }
4560        // The judges are finished; their checkouts are pure cost from here.
4561        let root = self.state.worktree_root();
4562        for j in 1..=self.roles.judges.len() {
4563            let wt = root.join(format!("judge-{j}"));
4564            if wt.exists() {
4565                remove_preserved_worktree(&repo, &wt).await?;
4566            }
4567        }
4568        // The design-deliberation stage is finished by the time a tally
4569        // exists — same reasoning as the judges above.
4570        if self.state.config.graph.advise {
4571            for k in 1..=self.state.config.graph.advisors {
4572                let wt = root.join(format!("advisor-{k}"));
4573                if wt.exists() {
4574                    remove_preserved_worktree(&repo, &wt).await?;
4575                }
4576            }
4577        }
4578        if !folded.is_empty() {
4579            self.state
4580                .event("fold", format!("folded candidates {}", folded.join(", ")));
4581            self.state.save()?;
4582        }
4583        Ok(())
4584    }
4585
4586    // ------------------------------------------------------------ base sync
4587
4588    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4589    /// anything verifies it.
4590    ///
4591    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4592    /// read whatever is checked out in the winner's worktree. Left alone that
4593    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4594    /// it when the run *branched* - and a run takes long enough that the base
4595    /// has usually moved by the time it gets here. A gate that ran there
4596    /// answers "green on the commit this run started from", not "green on
4597    /// what is about to land", and the difference showed up three times in
4598    /// one day as a green run whose merge would have reverted a file another
4599    /// pull request had already landed.
4600    ///
4601    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4602    /// `land::Step::Rebase` calls, rather than a second implementation of the
4603    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4604    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4605    /// doc warns about ("two answers to a question nobody notices until a
4606    /// diff is wrong").
4607    ///
4608    /// A conflict is not the end of the road: the standing rebase is handed
4609    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4610    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4611    /// land). Once it finishes, review and the gate run as usual on the
4612    /// rebased tree, which is where a breakage the new base caused is caught
4613    /// by the ordinary gate-fix round. magi resolves nothing itself.
4614    ///
4615    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4616    /// `state.base_sync.attempts`, is how many times the base is *rebased
4617    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4618    /// *conflict* was given to a fixer. When the fixer cannot finish the
4619    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4620    /// what was tried (rounds spent, paths still conflicted) and the branch
4621    /// and worktree stay exactly as they were - untouched, for a person to
4622    /// look at - which is also what makes re-entering this function
4623    /// afterwards a no-op instead of a second attempt at the same wall. A
4624    /// push failure ends the same way.
4625    async fn sync_to_base(&mut self) -> Result<()> {
4626        if self.state.status == RunStatus::AlreadyInBase {
4627            return Ok(());
4628        }
4629        let conflicted = self
4630            .state
4631            .base_sync
4632            .as_ref()
4633            .is_some_and(|s| s.conflict.is_some());
4634        let Some(winner) = self.state.winner().cloned() else {
4635            return Ok(());
4636        };
4637
4638        let repo = self.state.repo.clone();
4639        let remote = self.state.config.merge.remote.clone();
4640        let base_branch = self.state.base_branch.clone();
4641        let tracking = format!("{remote}/{base_branch}");
4642
4643        git::fetch(&repo, &remote, &base_branch).await.ok();
4644        // No network, or the remote never had this branch: the run already
4645        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4646        // otherwise), and one that got this far is not blocked by a fetch
4647        // that fails now. It just has no newer tip to compare against.
4648        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4649            return Ok(());
4650        };
4651
4652        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4653        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4654        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4655
4656        // Before any rebase, and before a recorded conflict is honoured: a
4657        // branch whose change reached the base under other commit ids has
4658        // nothing to rebase and nothing to conflict with, and a run that
4659        // already stopped on that phantom conflict recovers here on resume.
4660        // `behind == 0` with head == tip is a branch the base has since taken
4661        // in whole, whether or not a conflict was ever recorded: the ancestry
4662        // proof must still run (`classify` ignores a head still on the start
4663        // commit).
4664        if (behind > 0 || conflicted || head == tip)
4665            && self
4666                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4667                .await?
4668        {
4669            return Ok(());
4670        }
4671        if conflicted {
4672            return Ok(());
4673        }
4674
4675        if behind == 0 {
4676            // A fixer-finished rebase moves the branch ref before the
4677            // winner's worktree is told (`sync_to_head` below). A run that
4678            // died in between resumes here with `behind == 0` and a tree still
4679            // holding the pre-rebase files, which review and the gate would
4680            // then read. That state is exactly: HEAD moved off the tip the
4681            // rebase started from, yet the tree is still identical to that
4682            // tip. A tree with edits of its own differs from it, so nothing
4683            // is thrown away.
4684            if let Some(from) = self
4685                .state
4686                .rebase_fixes
4687                .iter()
4688                .rev()
4689                .find_map(|r| r.from.clone())
4690                && from != head
4691                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4692                    .await
4693                    .is_ok_and(|o| o.ok())
4694            {
4695                git::sync_to_head(&winner.worktree).await?;
4696            }
4697            // An earlier attempt may have rebased the branch locally and died
4698            // before pushing it (only the fresh-rebase arm below pushes).
4699            // Publish it now, so the plain push at PR time is not refused as
4700            // a non-fast-forward. A run already holding a recorded conflict
4701            // never reaches here; that case is out of scope.
4702            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4703            if let Some(why) = &conflict {
4704                self.state.status = RunStatus::Blocked;
4705                self.state.event("land", why.clone());
4706            }
4707            self.state.base_sync = Some(BaseSync {
4708                tip,
4709                behind: 0,
4710                attempts,
4711                conflict,
4712                already_in: None,
4713            });
4714            self.state.save()?;
4715            return Ok(());
4716        }
4717
4718        if attempts >= BASE_SYNC_ROUNDS {
4719            let why = format!(
4720                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4721                 rebase(s); rebasing again would only race it",
4722                winner.branch
4723            );
4724            self.state.status = RunStatus::Blocked;
4725            self.state.base_sync = Some(BaseSync {
4726                tip,
4727                behind,
4728                attempts,
4729                conflict: Some(why.clone()),
4730                already_in: None,
4731            });
4732            self.state.event("land", why);
4733            self.state.save()?;
4734            return Ok(());
4735        }
4736
4737        self.state.event(
4738            "land",
4739            format!(
4740                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4741                winner.branch
4742            ),
4743        );
4744        self.state.save()?;
4745
4746        // The remote's copy of the branch, read now and only if the fetch
4747        // really succeeded (a stale tracking ref must never pin a lease). It is
4748        // pushed over after a rebase only when it is a commit this branch
4749        // already contains, by ancestry or by patch (an earlier rebase of ours
4750        // that never reached the remote): anything else is somebody else's work.
4751        let branch_tracking = format!("{remote}/{}", winner.branch);
4752        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4753        let remote_tip = if self.state.task_pr_branch().is_none()
4754            && matches!(&fetched_branch, Ok(o) if o.ok())
4755        {
4756            git::rev_parse(&repo, &branch_tracking).await.ok()
4757        } else {
4758            None
4759        };
4760        // A remote tip this branch does not contain is somebody else's work:
4761        // rebasing would leave a local tip that can never be pushed. Stop
4762        // before touching anything and say so.
4763        if let Some(theirs) = &remote_tip
4764            && !git::is_ancestor(&repo, theirs, &head).await
4765            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4766                .await
4767                .is_ok_and(|missing| missing.is_empty())
4768        {
4769            let why = format!(
4770                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4771                 them",
4772                short(theirs),
4773                winner.branch
4774            );
4775            self.state.status = RunStatus::Blocked;
4776            self.state.base_sync = Some(BaseSync {
4777                tip,
4778                behind,
4779                attempts,
4780                conflict: Some(why.clone()),
4781                already_in: None,
4782            });
4783            self.state.event("land", why);
4784            self.state.save()?;
4785            return Ok(());
4786        }
4787
4788        let scratch = self.state.dir().join("base-sync");
4789        let rebased = match crate::rebase::rebase_with_fixer(
4790            &mut self.state,
4791            &scratch,
4792            &winner.branch,
4793            &tracking,
4794        )
4795        .await
4796        {
4797            Ok(crate::rebase::Rebased::Applied) => Ok(None),
4798            Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4799            Err(e) => Err(e),
4800        };
4801        let attempts = attempts + 1;
4802        match rebased {
4803            Ok(None) => {
4804                // The branch ref moved, but a worktree that already had it
4805                // checked out (the winner's) was not told; sync its index and
4806                // files before anything reads them.
4807                git::sync_to_head(&winner.worktree).await?;
4808                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4809                let mut conflict = None;
4810                if let Some(pinned) = &remote_tip {
4811                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4812                    match pushed {
4813                        Ok(o) if o.ok() => self.state.event(
4814                            "land",
4815                            format!("pushed rebased {} to {remote}", winner.branch),
4816                        ),
4817                        Ok(o) => {
4818                            conflict = Some(format!(
4819                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
4820                                winner.branch,
4821                                short(pinned),
4822                                o.stderr.chars().take(600).collect::<String>()
4823                            ));
4824                        }
4825                        Err(e) => {
4826                            conflict = Some(format!(
4827                                "rebased {} locally but could not push it: {e:#}",
4828                                winner.branch
4829                            ));
4830                        }
4831                    }
4832                }
4833                if let Some(why) = &conflict {
4834                    self.state.status = RunStatus::Blocked;
4835                    self.state.event("land", why.clone());
4836                }
4837                self.state.base_sync = Some(BaseSync {
4838                    tip: tip.clone(),
4839                    behind: 0,
4840                    attempts,
4841                    conflict,
4842                    already_in: None,
4843                });
4844                self.state
4845                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
4846            }
4847            Ok(Some(conflict)) => {
4848                let why = format!(
4849                    "{} conflicts with {tracking} and did not rebase: {}",
4850                    winner.branch,
4851                    conflict.chars().take(600).collect::<String>()
4852                );
4853                self.state.status = RunStatus::Blocked;
4854                self.state.base_sync = Some(BaseSync {
4855                    tip,
4856                    behind,
4857                    attempts,
4858                    conflict: Some(why.clone()),
4859                    already_in: None,
4860                });
4861                self.state.event("land", why);
4862            }
4863            // A refused Git write or a busy repository: nothing is wrong with
4864            // the branch. The scratch rebase stays standing and the run
4865            // resumes into it (`base_sync` must not claim a conflict).
4866            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4867                self.state.save()?;
4868                return Ok(());
4869            }
4870            Err(e) => {
4871                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4872                self.state.status = RunStatus::Blocked;
4873                self.state.base_sync = Some(BaseSync {
4874                    tip,
4875                    behind,
4876                    attempts,
4877                    conflict: Some(why.clone()),
4878                    already_in: None,
4879                });
4880                self.state.event("land", why);
4881            }
4882        }
4883        self.state.save()?;
4884        Ok(())
4885    }
4886
4887    /// Push a branch an earlier attempt rebased locally but never published,
4888    /// pinned to the remote tip read right after a successful fetch. Returns
4889    /// the reason when the run must stop; `None` when there was nothing to do
4890    /// (no remote copy, the same tip, or a remote copy this branch already
4891    /// contains, which the PR-time push fast-forwards) or the push succeeded.
4892    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4893        // Task competitions publish only after the PR text gate, to their
4894        // public head; never resurrect a candidate branch on the remote.
4895        if self.state.task_pr_branch().is_some() {
4896            return None;
4897        }
4898        let repo = self.state.repo.clone();
4899        let remote = self.state.config.merge.remote.clone();
4900        let fetched = git::fetch(&repo, &remote, branch).await;
4901        if !matches!(&fetched, Ok(o) if o.ok()) {
4902            return None;
4903        }
4904        let branch_tracking = format!("{remote}/{branch}");
4905        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4906        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4907            return None;
4908        }
4909        if !crate::reconcile::origin_missing(&repo, head, &theirs)
4910            .await
4911            .is_ok_and(|missing| missing.is_empty())
4912        {
4913            return Some(format!(
4914                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4915                 them",
4916                short(&theirs)
4917            ));
4918        }
4919        match git::push_pinned(&repo, &remote, branch, &theirs).await {
4920            Ok(o) if o.ok() => {
4921                self.state
4922                    .event("land", format!("pushed rebased {branch} to {remote}"));
4923                None
4924            }
4925            Ok(o) => Some(format!(
4926                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4927                 someone may have pushed): {}",
4928                short(&theirs),
4929                o.stderr.chars().take(600).collect::<String>()
4930            )),
4931            Err(e) => Some(format!(
4932                "{branch} is rebased locally but could not be pushed: {e:#}"
4933            )),
4934        }
4935    }
4936
4937    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
4938    /// is already on `tip` under other commit ids ([`crate::already`]); returns
4939    /// whether it did.
4940    ///
4941    /// Checked only when the base is ahead of the branch. A failing check is
4942    /// "not proven" - the ordinary rebase path then decides - never a reason to
4943    /// stop the run.
4944    ///
4945    /// The remote copy of the branch is held to the same standard as the local
4946    /// one: if it carries a tip this worktree does not, that tip must itself be
4947    /// proven in the base, or nothing is settled (a pull request would
4948    /// otherwise be closed over commits nobody checked). The pull request is
4949    /// closed *before* the terminal status is saved; if that fails for a
4950    /// reason other than a refusal (no network, a `gh` error) the run is left
4951    /// `Blocked` with the reason as its conflict, which a resume retries -
4952    /// the same recovery a phantom conflict gets.
4953    async fn settle_already_in(
4954        &mut self,
4955        branch: &str,
4956        tip: &str,
4957        head: &str,
4958        attempts: usize,
4959        behind: usize,
4960    ) -> Result<bool> {
4961        let repo = self.state.repo.clone();
4962        let remote = self.state.config.merge.remote.clone();
4963        let start = self.state.base_commit.clone();
4964        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4965            Ok(Some(e)) => e,
4966            Ok(None) => return Ok(false),
4967            Err(e) => {
4968                tracing::warn!("already-in-base check for {branch}: {e:#}");
4969                return Ok(false);
4970            }
4971        };
4972        let mut verified = vec![head.to_owned()];
4973        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4974        let fetched = git::fetch(&repo, &remote, &public_branch).await;
4975        if matches!(&fetched, Ok(o) if o.ok())
4976            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4977            && theirs != head
4978        {
4979            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4980                Ok(Some(_)) => verified.push(theirs),
4981                _ => return Ok(false),
4982            }
4983        }
4984        let base_branch = self.state.base_branch.clone();
4985        let message = format!(
4986            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4987             land",
4988            evidence.names(),
4989            evidence.proof.as_str()
4990        );
4991        let closed =
4992            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4993                .await;
4994        match closed {
4995            Ok(Ok(url)) => self
4996                .state
4997                .event("land", format!("closed {url}: superseded on {base_branch}")),
4998            Ok(Err(why)) => self
4999                .state
5000                .event("land", format!("did not close a pull request: {why}")),
5001            Err(e) => {
5002                let why = format!(
5003                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
5004                     not be closed ({e:#}); resume to retry"
5005                );
5006                self.state.status = RunStatus::Blocked;
5007                self.state.base_sync = Some(BaseSync {
5008                    tip: tip.to_owned(),
5009                    behind,
5010                    attempts,
5011                    conflict: Some(why.clone()),
5012                    already_in: None,
5013                });
5014                self.state.event("land", why);
5015                self.state.save()?;
5016                return Ok(true);
5017            }
5018        }
5019        self.state.status = RunStatus::AlreadyInBase;
5020        self.state.base_sync = Some(BaseSync {
5021            tip: tip.to_owned(),
5022            behind,
5023            attempts,
5024            conflict: None,
5025            already_in: Some(evidence),
5026        });
5027        self.state.event("land", message);
5028        self.state.save()?;
5029        self.settle_questions();
5030        Ok(true)
5031    }
5032
5033    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
5034    /// last landed the winner on, once it has run, else the commit the run
5035    /// branched from.
5036    ///
5037    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
5038    /// and `vote` all happen before there is a winner to rebase, so they
5039    /// compare every candidate against the branch point on purpose, and a
5040    /// base that moves after they are already done cannot change an answer
5041    /// they already gave.
5042    fn landing_base(&self) -> String {
5043        self.state
5044            .base_sync
5045            .as_ref()
5046            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5047    }
5048
5049    // ------------------------------------------------------- operator fix
5050
5051    /// Route specific, already-recorded review findings to a fixer for a
5052    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5053    /// entry point.
5054    ///
5055    /// Distinct from `review_loop`'s own fix step in three ways: it never
5056    /// runs a reviewer wave, it never spends review-round budget, and what
5057    /// happened is recorded as an [`OperatorFixRequest`] appended to
5058    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5059    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5060    /// and vote must never be rewritten to look like a manufactured blocking
5061    /// verdict.
5062    ///
5063    /// Only meaningful once review has actually concluded: `Ready` (handed
5064    /// off with findings still open, or simply concluded clean while minor
5065    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5066    /// gate failed). Everything else is refused: a run still in progress
5067    /// should simply be resumed, and a `Merged` run's branch has already
5068    /// landed — reopening *this* run's own record cannot change that, so the
5069    /// answer there is a fresh `magi review <branch>`.
5070    ///
5071    /// A real commit here re-verifies through a fresh, ordinary review-only
5072    /// run on the same branch ([`Self::review`]) rather than reopening this
5073    /// run's own `review_loop`: once any round in this run's history went
5074    /// clean, `review_conclusion` treats that as permanent by design (the
5075    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5076    /// way to force one more genuine reviewer wave out of *this* run without
5077    /// either rewriting history or weakening that guarantee for every other
5078    /// caller. A review-only run costs nothing extra — no implementation, no
5079    /// judging, no vote — and exercises the exact same review → verify →
5080    /// gate → (human) merge path, unmodified.
5081    pub async fn fix_selected(
5082        &mut self,
5083        ids: &[String],
5084        reason: &str,
5085        allow_stale: bool,
5086    ) -> Result<()> {
5087        let reason = reason.trim();
5088        if reason.is_empty() {
5089            bail!("a fix request needs a reason — that is the operator's own record of why");
5090        }
5091        if ids.is_empty() {
5092            bail!("no finding id given");
5093        }
5094        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5095            bail!(
5096                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5097                 has already concluded — can be given a targeted fix. A run still \
5098                 in progress should simply be resumed; a `merged` run's branch has \
5099                 already landed, so its answer is a fresh `magi review <branch>`, \
5100                 not reopening this run's own record",
5101                self.state.id,
5102                self.state.status.as_str()
5103            );
5104        }
5105        let Some(winner) = self.state.winner().cloned() else {
5106            bail!("run {} has no winning candidate to fix", self.state.id);
5107        };
5108        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5109            bail!(
5110                "branch `{}` no longer exists; this run cannot be extended",
5111                winner.branch
5112            );
5113        }
5114        let home = crate::run::home();
5115        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5116            bail!(
5117                "run {} is currently being worked on by another magi process",
5118                self.state.id
5119            );
5120        }
5121        // Held for the rest of this call, including the follow-up review
5122        // below: two `magi fix` invocations against the same run must not
5123        // both reach the worktree manipulation further down, which would
5124        // otherwise race to remove and recreate the same directory — see
5125        // [`FixClaim`]'s own doc.
5126        let _claim = FixClaim::acquire(&self.state.dir())?;
5127
5128        // Resolve every id before spending anything — an unknown id refuses
5129        // the whole request rather than silently dropping it — and dedup
5130        // while keeping the operator's own order.
5131        let mut seen = BTreeSet::new();
5132        let mut findings = Vec::new();
5133        let mut missing = Vec::new();
5134        for id in ids {
5135            if !seen.insert(id.clone()) {
5136                continue;
5137            }
5138            match self.state.finding(id) {
5139                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5140                    id: f.id.clone(),
5141                    severity: f.severity,
5142                    reviewer_vote: rec.vote,
5143                    round: round.round,
5144                    round_head: round.head.clone(),
5145                    reviewer: rec.reviewer,
5146                    agent: rec.agent.clone(),
5147                    file: f.file.clone(),
5148                    line: f.line,
5149                    title: f.title.clone(),
5150                    detail: f.detail.clone(),
5151                    outcome: OperatorFixOutcome::Pending,
5152                }),
5153                None => missing.push(id.clone()),
5154            }
5155        }
5156        if !missing.is_empty() {
5157            bail!(
5158                "unknown finding id(s): {}; nothing was changed",
5159                missing.join(", ")
5160            );
5161        }
5162
5163        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5164        let stale_details: Vec<(String, String)> = findings
5165            .iter()
5166            .filter(|f| f.round_head != head_at_request)
5167            .map(|f| (f.id.clone(), f.round_head.clone()))
5168            .collect();
5169        let stale = !stale_details.is_empty();
5170        if stale && !allow_stale {
5171            bail!(
5172                "the branch has moved since some finding(s) were raised — {} — now \
5173                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5174                stale_details
5175                    .iter()
5176                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5177                    .collect::<Vec<_>>()
5178                    .join(", "),
5179                short(&head_at_request)
5180            );
5181        }
5182
5183        let request = OperatorFixRequest {
5184            requested_at: Timestamp::now(),
5185            reason: reason.to_owned(),
5186            findings,
5187            head_at_request: head_at_request.clone(),
5188            allow_stale,
5189            stale,
5190            fix: None,
5191            result_head: None,
5192            follow_up_review_run: None,
5193        };
5194        self.state.event(
5195            "fix",
5196            format!(
5197                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5198                request.findings.len(),
5199                request
5200                    .findings
5201                    .iter()
5202                    .map(|f| f.id.as_str())
5203                    .collect::<Vec<_>>()
5204                    .join(", "),
5205            ),
5206        );
5207        // Recorded now, before any worktree work or the fixer call itself —
5208        // and re-saved at each checkpoint below: a crash at any point after
5209        // this (mid fixer call, mid follow-up review) must not lose the fact
5210        // that this was requested, for which findings, and why. Everything
5211        // past this point reads and writes through `request_index` rather
5212        // than a local variable, since `request` itself is moved here.
5213        self.state.operator_fixes.push(request);
5214        self.state.save()?;
5215        let request_index = self.state.operator_fixes.len() - 1;
5216
5217        // A fresh, dedicated worktree for this one call, never the winner's
5218        // own worktree in place: that one may already be gone (folded away),
5219        // and reusing it in place would leave the branch checked out there
5220        // when the follow-up review below tries to check it out again. Freed
5221        // immediately after, either way — but only once confirmed clean:
5222        // `worktree_remove` is a `git worktree remove --force`, which would
5223        // otherwise discard uncommitted work left there by the operator or
5224        // another process before this had a chance to even look at it.
5225        let repo = self.state.repo.clone();
5226        let coordination = self.coordinate(&repo).await?;
5227        if winner.worktree.exists() {
5228            // Lockfiles a rescue commit withheld stay untracked on purpose and
5229            // are already recorded; they are not the operator's work to protect.
5230            let dirty = git::git(
5231                &winner.worktree,
5232                &["status", "--porcelain", "--untracked-files=all"],
5233            )
5234            .await?;
5235            let only_withheld = dirty.lines().all(|l| {
5236                l.strip_prefix("?? ")
5237                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5238            });
5239            if !only_withheld {
5240                bail!(
5241                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5242                     discard them first",
5243                    winner.worktree.display()
5244                );
5245            }
5246            git::worktree_remove(&self.state.repo, &winner.worktree)
5247                .await
5248                .ok();
5249        }
5250        let fix_worktree = self.state.worktree_root().join("operator-fix");
5251        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5252        git::git(
5253            &self.state.repo,
5254            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5255        )
5256        .await
5257        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5258        if !git::is_clean(&fix_worktree).await? {
5259            git::worktree_remove(&self.state.repo, &fix_worktree)
5260                .await
5261                .ok();
5262            bail!(
5263                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5264                winner.branch
5265            );
5266        }
5267        let repo_for_setup = self.state.repo.clone();
5268        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5269            Ok(()) => {
5270                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5271            }
5272            Err(e) => Err(e),
5273        };
5274        if let Err(e) = setup {
5275            git::worktree_remove(&self.state.repo, &fix_worktree)
5276                .await
5277                .ok();
5278            return Err(e);
5279        }
5280
5281        let run_id = self.state.id.clone();
5282        let prompts = self.state.config.prompts.clone();
5283        let language = self.state.config.graph.language.clone();
5284        let sessions = self.state.config.graph.sessions;
5285        let artifacts = agent::artifacts_dir(&self.state.dir());
5286        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5287            .findings
5288            .iter()
5289            .map(|f| Finding {
5290                id: f.id.clone(),
5291                severity: f.severity,
5292                file: f.file.clone(),
5293                line: f.line,
5294                title: f.title.clone(),
5295                detail: f.detail.clone(),
5296            })
5297            .collect();
5298        let fix_prompt = prompt::operator_fix_in(
5299            &self.state.instruction,
5300            &finding_list,
5301            reason,
5302            &stale_details,
5303            &head_at_request,
5304            &language,
5305            &self.state.config.graph.github_language,
5306        );
5307        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5308        drop(coordination);
5309        let (job, seat, out) = self
5310            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5311                prompt: fix_prompt.clone(),
5312                spec,
5313                seat,
5314                cwd: fix_worktree.clone(),
5315                timeout,
5316                allow_write: true,
5317                sessions,
5318                artifacts: artifacts.clone(),
5319                stem: "operator-fix".to_owned(),
5320                handover: None,
5321            })
5322            .await;
5323        let agent_id = seat.agent.clone();
5324
5325        let mut fix = FixRecord {
5326            agent: agent_id,
5327            addressed: Vec::new(),
5328            rejected: Vec::new(),
5329            notes: String::new(),
5330            committed: false,
5331            failed: None,
5332            duration_ms: 0,
5333            continuation: None,
5334        };
5335        let mut final_seat = seat.clone();
5336        match out {
5337            AgentOutcome::Ok(o) => {
5338                fix.duration_ms = o.duration_ms;
5339                let parsed = verdict::extract_json::<FixReport>(&o.text);
5340                let incomplete_reason = match &parsed {
5341                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5342                        "the reply parsed, but it reported a command whose own CLI \
5343                         never confirmed an exit status"
5344                            .to_owned(),
5345                    ),
5346                    Ok(_) => None,
5347                    Err(e) => Some(e.to_string()),
5348                };
5349                match incomplete_reason {
5350                    None => {
5351                        let report = parsed.expect("checked Ok above");
5352                        fix.addressed = report.addressed;
5353                        fix.rejected = report.rejected;
5354                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5355                    }
5356                    Some(reason) => {
5357                        let (resumed_seat, resolved, failure, cont) = self
5358                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5359                            .await;
5360                        fix.duration_ms += cont.cumulative_wait_ms;
5361                        fix.continuation = Some(cont);
5362                        final_seat = resumed_seat;
5363                        match resolved {
5364                            Some(report) => {
5365                                fix.addressed = report.addressed;
5366                                fix.rejected = report.rejected;
5367                                fix.notes =
5368                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5369                            }
5370                            None => fix.failed = failure,
5371                        }
5372                    }
5373                }
5374            }
5375            AgentOutcome::Dropped(o) => {
5376                fix.duration_ms = o.duration_ms;
5377                let why = o
5378                    .dropped
5379                    .as_ref()
5380                    .map(|d| d.why.as_str())
5381                    .unwrap_or("the CLI ended the stream without delivering its answer");
5382                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5383            }
5384            AgentOutcome::Quota(o) => {
5385                self.state.quota.push(QuotaLoss {
5386                    seat: final_seat.key.clone(),
5387                    node: "fix".to_owned(),
5388                    at: Timestamp::now(),
5389                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5390                });
5391                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5392            }
5393            AgentOutcome::Failed(e) => fix.failed = Some(e),
5394            AgentOutcome::ResourceBlocked(e) => {
5395                fix.failed = Some(format!("the fixer was not started: {e}"));
5396            }
5397        }
5398        if fix.continuation.is_none() {
5399            fix.continuation = Some(ContinuationRecord::not_needed());
5400        }
5401        self.state.seats.insert(final_seat.key.clone(), final_seat);
5402
5403        let rescue_message = format!(
5404            "magi: operator-selected fix ({}) (uncommitted work)",
5405            self.state.operator_fixes[request_index]
5406                .findings
5407                .iter()
5408                .map(|f| f.id.as_str())
5409                .collect::<Vec<_>>()
5410                .join(", ")
5411        );
5412        let repo = self.state.repo.clone();
5413        let coordination = self.coordinate(&repo).await?;
5414        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5415            self.state.note_withheld("fix", &r.withheld);
5416        }
5417        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5418        fix.committed = after != head_at_request;
5419        git::worktree_remove(&self.state.repo, &fix_worktree)
5420            .await
5421            .ok();
5422        drop(coordination);
5423
5424        self.state.event(
5425            "fix",
5426            match &fix.failed {
5427                Some(reason) => format!(
5428                    "operator fix: adoption report was lost ({reason}); {}",
5429                    if fix.committed {
5430                        "committed"
5431                    } else {
5432                        "NO new commit"
5433                    }
5434                ),
5435                None => format!(
5436                    "operator fix: {} addressed, {} rejected, {}",
5437                    fix.addressed.len(),
5438                    fix.rejected.len(),
5439                    if fix.committed {
5440                        "committed"
5441                    } else {
5442                        "NO new commit"
5443                    }
5444                ),
5445            },
5446        );
5447
5448        // Every selected finding gets an outcome — never left `Pending` once
5449        // the fixer's own turn is over. A report that never came back at all
5450        // marks every one of them `Unreported`, not silently "not addressed":
5451        // quota, a dropped stream, or an exhausted continuation are gaps in
5452        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5453        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5454        for f in &mut self.state.operator_fixes[request_index].findings {
5455            f.outcome = if fix.failed.is_some() {
5456                OperatorFixOutcome::Unreported
5457            } else if fix.addressed.contains(&f.id) {
5458                OperatorFixOutcome::Addressed
5459            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5460                OperatorFixOutcome::Rejected { why: r.why.clone() }
5461            } else {
5462                OperatorFixOutcome::Unreported
5463            };
5464        }
5465
5466        let committed = fix.committed;
5467        if committed {
5468            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5469        }
5470        self.state.operator_fixes[request_index].fix = Some(fix);
5471        // Saved again now that the fixer's own outcome is final, on top of
5472        // the save right after the request was first pushed above.
5473        self.state.save()?;
5474
5475        if committed {
5476            self.state.event(
5477                "fix",
5478                format!(
5479                    "operator fix committed {}; opening a follow-up review-only run",
5480                    short(&after)
5481                ),
5482            );
5483            // The operator asked for the fix, and the follow-up serves whatever
5484            // task the run it follows served.
5485            let origin =
5486                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5487            match Self::review(
5488                &self.state.repo,
5489                &winner.branch,
5490                self.state.config.clone(),
5491                origin,
5492            )
5493            .await
5494            {
5495                Ok(mut follow_up) => {
5496                    follow_up.state.event(
5497                        "start",
5498                        format!(
5499                            "requested by an operator fix on run {} for finding(s) {}",
5500                            self.state.id,
5501                            self.state.operator_fixes[request_index]
5502                                .findings
5503                                .iter()
5504                                .map(|f| f.id.as_str())
5505                                .collect::<Vec<_>>()
5506                                .join(", "),
5507                        ),
5508                    );
5509                    follow_up.state.save()?;
5510                    let follow_up_id = follow_up.state.id.clone();
5511                    // The follow-up is a run like any other: it belongs to the
5512                    // task of the run it follows, or to one filed for it.
5513                    let adopted = match crate::direct::adopt(
5514                        &follow_up.state,
5515                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5516                    ) {
5517                        Ok(a) => a,
5518                        Err(e) => {
5519                            // An ownerless run must not spend agent calls; it
5520                            // stays saved, and `magi run --resume` adopts it.
5521                            self.state.event(
5522                                "fix",
5523                                format!(
5524                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5525                                ),
5526                            );
5527                            self.state.save()?;
5528                            return Ok(());
5529                        }
5530                    };
5531                    let executed = follow_up.execute().await;
5532                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5533                    if let Some(a) = adopted {
5534                        a.finish(&follow_up.state, executed);
5535                    }
5536                    if let Some(e) = failure {
5537                        self.state.event(
5538                            "fix",
5539                            format!(
5540                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5541                            ),
5542                        );
5543                    }
5544                    self.state.operator_fixes[request_index].follow_up_review_run =
5545                        Some(follow_up_id);
5546                }
5547                Err(e) => {
5548                    self.state.event(
5549                        "fix",
5550                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5551                    );
5552                }
5553            }
5554            self.state.save()?;
5555        }
5556
5557        Ok(())
5558    }
5559
5560    // --------------------------------------------------------------- review
5561
5562    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5563    /// names, in order, each at most once, moving on only when the call
5564    /// advances (an error, a quota hit or nothing usable - the same decision
5565    /// point `agent::chain_advances` is for the other chained roles).
5566    ///
5567    /// Each agent gets its own seat from `Runner::seat`: the same agent
5568    /// continues its conversation (the winner's own implementer seat when it
5569    /// is the winner's author, now that the competition is over), another
5570    /// takes a fresh one so the full prompt is sent again. A handover is
5571    /// recorded under `node`, which is what makes the fallback stick for the
5572    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5573    /// is returned, so an exhausted chain reads exactly like a single failed
5574    /// fixer: one quota loss, the same wording, the same refund. An earlier
5575    /// attempt's edits are left in the tree and judged, with the final
5576    /// attempt's, by what git says afterwards.
5577    ///
5578    /// Returns the job that produced the outcome, for `continue_fix_report`.
5579    async fn ask_fixer(
5580        &mut self,
5581        winner: &Candidate,
5582        node: &'static str,
5583        round: Option<usize>,
5584        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5585    ) -> (SeatJob, SeatState, AgentOutcome) {
5586        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5587        self.ask_fixer_among(attempts, node, round, None, build)
5588            .await
5589    }
5590
5591    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5592    /// worktree and a commit: before the next agent is asked after one that
5593    /// did not satisfy the call, the worktree is put back on that commit, so
5594    /// a failed agent's partial edits are not carried into the next one.
5595    async fn ask_fixer_among(
5596        &mut self,
5597        attempts: Vec<(AgentSpec, String)>,
5598        node: &'static str,
5599        round: Option<usize>,
5600        restore: Option<(&Path, &str)>,
5601        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5602    ) -> (SeatJob, SeatState, AgentOutcome) {
5603        let run_id = self.state.id.clone();
5604        let prompts = self.state.config.prompts.clone();
5605        let cache = self.state.config.cache_dir();
5606        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5607        let mut last = None;
5608        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5609            let seat = self.seat(&key, &spec.id);
5610            let mut job = build(spec.clone(), seat);
5611            if i > 0 {
5612                job.stem = format!("{}-{}", job.stem, spec.id);
5613            }
5614            let ctx = WaveCtx {
5615                carry_seats: false,
5616                pauses: Some((&self.pause, &self.interrupt)),
5617                run: &run_id,
5618                node,
5619                prompts: &prompts,
5620                cache: cache.as_deref(),
5621                round,
5622            };
5623            let (seat, out) =
5624                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5625            // A resource block never advances the chain: the chosen agent
5626            // was not asked, so it must be the one asked again on resume.
5627            let advances = match &out {
5628                AgentOutcome::Ok(o) => agent::output_advances(o),
5629                AgentOutcome::ResourceBlocked(_) => false,
5630                _ => true,
5631            };
5632            if let Some(next) = ids.get(i + 1)
5633                && advances
5634            {
5635                self.state.seats.insert(seat.key.clone(), seat.clone());
5636                let class =
5637                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5638                let reason = match &out {
5639                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5640                    other => fail_reason(other),
5641                };
5642                // A failed agent's edits must not reach the next one, and a
5643                // restore that did not work ends the chain: the call fails
5644                // and the caller's own (confirmed) restore decides.
5645                if let Some((wt, rev)) = restore
5646                    && let Err(e) = self.graft_restore(wt, rev).await
5647                {
5648                    let why =
5649                        format!("could not restore the worktree before the next fixer: {e:#}");
5650                    tracing::warn!("{why}");
5651                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5652                        AgentOutcome::ResourceBlocked(why)
5653                    } else {
5654                        AgentOutcome::Failed(why)
5655                    };
5656                    last = Some((job, seat, out));
5657                    break;
5658                }
5659                record_handover(
5660                    &mut self.state,
5661                    node,
5662                    &seat.key,
5663                    &spec.id,
5664                    next,
5665                    &class,
5666                    &reason,
5667                );
5668                continue;
5669            }
5670            last = Some((job, seat, out));
5671            break;
5672        }
5673        last.expect("the fixer chain always has an entry")
5674    }
5675
5676    /// The one repair a verification-only failure earns (see
5677    /// [`crate::recovery`]), before the first review round of a run that took
5678    /// a branch over.
5679    ///
5680    /// Saved as started *before* the fixer is called, so a resume never
5681    /// repairs twice; a start found on re-entry is rolled back to the commit
5682    /// it began from. A failed or unusable fixer leaves the tree as it was and
5683    /// the review goes on over it, so the first round's own verification
5684    /// reports the failure again. The failure is never carried into a round as
5685    /// a result: verification is re-run on whatever commit this leaves.
5686    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5687        use crate::recovery::{Mode, Repair};
5688        if !self.state.reviews.is_empty() {
5689            return Ok(());
5690        }
5691        let Some(mut rec) = self.state.recovery.clone() else {
5692            return Ok(());
5693        };
5694        if rec.mode != Mode::VerifyRepair {
5695            return Ok(());
5696        }
5697        let wt = winner.worktree.clone();
5698        let mut head = git::rev_parse(&wt, "HEAD").await?;
5699        match rec.repair.clone() {
5700            Repair::Done { .. } => return Ok(()),
5701            Repair::WaitingForRestore { before } => {
5702                self.graft_restore(&wt, &before).await?;
5703                head = before;
5704                rec.repair = Repair::NotStarted;
5705                self.state.recovery = Some(rec.clone());
5706                self.state.save()?;
5707            }
5708            Repair::Started { before } => {
5709                // Interrupted: the unfinished edits are not trusted.
5710                // An unconfirmed rollback blocks the run with `Started` kept,
5711                // so a resume retries it; the tree is never reviewed as is.
5712                if let Err(e) = self.graft_restore(&wt, &before).await {
5713                    if e.is::<crate::repo_lock::ResourceBlocked>() {
5714                        return Err(e);
5715                    }
5716                    return self
5717                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5718                        .await;
5719                }
5720                rec.repair = Repair::Done {
5721                    before: before.clone(),
5722                    after: before,
5723                    committed: false,
5724                    failed: Some("interrupted; rolled back".to_owned()),
5725                };
5726                self.state
5727                    .event("recovery", "the repair was interrupted and rolled back");
5728                self.state.recovery = Some(rec);
5729                self.state.save()?;
5730                return Ok(());
5731            }
5732            Repair::NotStarted => {}
5733        }
5734        if crate::recovery::recheck(&mut rec, &head) {
5735            self.state.event("recovery", rec.summary());
5736            self.state.recovery = Some(rec);
5737            self.state.save()?;
5738            return Ok(());
5739        }
5740        rec.repair = Repair::Started {
5741            before: head.clone(),
5742        };
5743        self.state.recovery = Some(rec.clone());
5744        self.state.save()?;
5745
5746        let language = self.state.config.graph.language.clone();
5747        let sessions = self.state.config.graph.sessions;
5748        let artifacts = agent::artifacts_dir(&self.state.dir());
5749        let fix_prompt = prompt::recovery_fix(
5750            &self.state.instruction,
5751            &rec,
5752            self.state.config.graph.review_rounds,
5753            &language,
5754        );
5755        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5756        self.state.event(
5757            "recovery",
5758            format!(
5759                "repair: the fixer starts from the failed verification on {}",
5760                short(&head)
5761            ),
5762        );
5763        let (_job, seat, out) = self
5764            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5765                prompt: fix_prompt.clone(),
5766                spec,
5767                seat,
5768                cwd: wt.clone(),
5769                timeout,
5770                allow_write: true,
5771                sessions,
5772                artifacts: artifacts.clone(),
5773                stem: "fix-recovery".to_owned(),
5774                handover: None,
5775            })
5776            .await;
5777        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5778            rec.repair = Repair::WaitingForRestore { before: head };
5779            self.state.recovery = Some(rec);
5780            return self.stop_for_resource();
5781        }
5782        let failed = match &out {
5783            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5784            AgentOutcome::Quota(o) => {
5785                self.state.quota.push(QuotaLoss {
5786                    seat: seat.key.clone(),
5787                    node: "fix".to_owned(),
5788                    at: Timestamp::now(),
5789                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5790                });
5791                Some("rate limited (quota); the fixer could not run".to_owned())
5792            }
5793            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5794            AgentOutcome::Failed(e) => Some(e.clone()),
5795        };
5796        self.state.seats.insert(seat.key.clone(), seat);
5797        let coordination = self.coordinate(&wt).await?;
5798        let failed = match failed {
5799            // The agent's partial edits are not kept.
5800            Some(why) => {
5801                if let Err(e) = git::restore_to(&wt, &head).await {
5802                    return self
5803                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5804                        .await;
5805                }
5806                Some(why)
5807            }
5808            None => {
5809                if let Ok(r) = git::rescue_commit(
5810                    &wt,
5811                    "magi: recovery repair of the failed verification (uncommitted work)",
5812                )
5813                .await
5814                {
5815                    self.state.note_withheld("fix", &r.withheld);
5816                }
5817                None
5818            }
5819        };
5820        let after = git::rev_parse(&wt, "HEAD").await?;
5821        drop(coordination);
5822        self.state.event(
5823            "recovery",
5824            match &failed {
5825                Some(why) => format!("repair: nothing kept ({why})"),
5826                None if after != head => {
5827                    format!("repair: committed {}", short(&after))
5828                }
5829                None => "repair: the fixer made no commit".to_owned(),
5830            },
5831        );
5832        rec.repair = Repair::Done {
5833            before: head.clone(),
5834            committed: after != head,
5835            after,
5836            failed,
5837        };
5838        self.state.recovery = Some(rec);
5839        self.state.save()?;
5840        Ok(())
5841    }
5842
5843    /// A repair whose rollback could not be confirmed: the run is blocked and
5844    /// resumable, and the unconfirmed state is never reviewed.
5845    async fn stop_recovery(
5846        &mut self,
5847        mut rec: crate::recovery::RecoveryRecord,
5848        head: &str,
5849        why: String,
5850    ) -> Result<()> {
5851        // `Started` stays recorded: a resume retries the restore.
5852        rec.repair = crate::recovery::Repair::Started {
5853            before: head.to_owned(),
5854        };
5855        self.state.recovery = Some(rec);
5856        self.state
5857            .event("recovery", format!("repair could not be undone: {why}"));
5858        self.state.status = RunStatus::Blocked;
5859        self.state.save()?;
5860        bail!("recovery repair could not be rolled back: {why}")
5861    }
5862
5863    async fn review_loop(&mut self) -> Result<()> {
5864        loop {
5865            if self
5866                .state
5867                .base_sync
5868                .as_ref()
5869                .is_some_and(|s| s.conflict.is_some())
5870            {
5871                return Ok(());
5872            }
5873            if self.catchup_repairable() {
5874                let record = self.state.reviews.last().expect("repairable round").clone();
5875                let winner = self
5876                    .state
5877                    .winner()
5878                    .cloned()
5879                    .context("catch-up repair has no winner")?;
5880                if !self.fix_review_round(&winner, record).await? {
5881                    return Ok(());
5882                }
5883            }
5884            self.review_loop_pass().await?;
5885            if !self.catchup_repairable() {
5886                return Ok(());
5887            }
5888        }
5889    }
5890
5891    /// Derive an owed catch-up repair from durable round evidence, including
5892    /// reentry after a crash before the fixer was dispatched.
5893    fn catchup_repairable(&self) -> bool {
5894        self.state.reviews.len() < self.state.config.graph.review_rounds
5895            && self.state.reviews.last().is_some_and(|r| {
5896                !r.clean
5897                    && r.blocking == 0
5898                    && r.fix.is_none()
5899                    && r.e2e_status() == E2eStatus::Failed
5900            })
5901    }
5902
5903    async fn review_loop_pass(&mut self) -> Result<()> {
5904        // A base that would not rebase is a person's decision, not a review
5905        // round: nothing here would change the answer, and reviewers and a
5906        // fixer would be spending real budget on a tree that cannot land
5907        // regardless of what they find.
5908        if self
5909            .state
5910            .base_sync
5911            .as_ref()
5912            .is_some_and(|s| s.conflict.is_some())
5913        {
5914            return Ok(());
5915        }
5916        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
5917        // agent files with `magi task add` name the run that paid for it. The
5918        // prompt overlay is cloned alongside it because the waves borrow it
5919        // while `self` is mutably borrowed by the node's own bookkeeping.
5920        let run_id = self.state.id.clone();
5921        let prompts = self.state.config.prompts.clone();
5922        let Some(winner) = self.state.winner().cloned() else {
5923            return Ok(());
5924        };
5925        let max_rounds = self.state.config.graph.review_rounds;
5926        // A clean round, an exhausted round budget, or a stalled tree (see
5927        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
5928        // they are recorded — recomputed here, not read off `status`, so a
5929        // reentry into a run that already stopped restates the identical
5930        // verdict instead of silently handing back whatever an earlier node
5931        // in this same walk clobbered `status` to (a solo-candidate
5932        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
5933        // below runs an empty range once the budget is spent, and would
5934        // otherwise fall through without touching `status` at all.
5935        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5936            // A reentry after a crash between the last round's save and
5937            // `stop_reviewing` reaches the hand-off here, not there.
5938            if status == RunStatus::Gating {
5939                self.record_contested_handoff();
5940            }
5941            self.state.status = status;
5942            self.state.save()?;
5943            return Ok(());
5944        }
5945        self.state.status = RunStatus::Reviewing;
5946        // A last recorded round whose own verification never resolved
5947        // (`ResourceBlocked` — the shared build cache, not the patch) is
5948        // never a concluded round, whatever the round budget says: starting
5949        // a fresh round on top of it would spend a whole new reviewer wave
5950        // re-reading an unchanged patch instead of just retrying the one
5951        // check that actually needs it, and once the budget is spent the
5952        // loop below has nothing left to do at all (its range is empty).
5953        // Retry that check directly instead, exactly the same retry
5954        // `stop_reviewing` already does for its own catch-up case.
5955        if self.state.reviews.last().is_some_and(|r| {
5956            r.e2e_status() == E2eStatus::ResourceBlocked
5957                || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
5958        }) {
5959            let shell = self.state.config.shell();
5960            return self
5961                .stop_unresolved_review(
5962                    "the last round's own verification never resolved",
5963                    &shell,
5964                    &winner.worktree,
5965                )
5966                .await;
5967        }
5968
5969        self.recovery_repair(&winner).await?;
5970        if self.resource_halt()? {
5971            return Ok(());
5972        }
5973
5974        let repo = self.state.repo.clone();
5975        let root = self.state.worktree_root();
5976        let language = self.state.config.graph.language.clone();
5977        let sessions = self.state.config.graph.sessions;
5978        let artifacts = self.merge_repair_artifacts();
5979        let base = self.landing_base();
5980        let base_short = short(&base);
5981        let reviewers = self.roles.reviewers.clone();
5982        let shell = self.state.config.shell();
5983
5984        for round in (self.state.reviews.len() + 1)..=max_rounds {
5985            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5986            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5987            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5988            // The prior round's own record, already persisted — never a
5989            // hand-carried variable of just its failing output: that is
5990            // exactly what let a round's e2e result drift out of sync with
5991            // which commit it was actually about (see `SCHEMA`'s doc for
5992            // schema 8). Judged against `head`, the commit reviewers are
5993            // about to look at now, so the summary always reads as "an
5994            // earlier head" here — this round's own patch has not been
5995            // checked yet.
5996            let prev_verification = self
5997                .state
5998                .reviews
5999                .last()
6000                .and_then(|r| r.verification_summary(&head));
6001            // Only the first round of a run that took a branch over: later
6002            // rounds have this run's own history.
6003            let prior_section = match (&self.state.recovery, round) {
6004                (Some(rec), 1) if self.state.reviews.is_empty() => {
6005                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6006                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6007                            Some(crate::recovery::head_of(
6008                                &git::diff(&winner.worktree, reviewed, "HEAD")
6009                                    .await
6010                                    .unwrap_or_default(),
6011                                crate::recovery::SINCE_DIFF_MAX,
6012                            ))
6013                        }
6014                        _ => None,
6015                    };
6016                    Some(prompt::recovery_review(rec, since.as_deref()))
6017                }
6018                _ => None,
6019            };
6020
6021            // Each reviewer gets its own detached checkout of exactly this
6022            // commit: nobody can perturb the winner's tree, and the fixer can
6023            // keep working without racing a reviewer.
6024            let mut jobs = Vec::new();
6025            let coordination = self.coordinate(&repo).await?;
6026            for (r, spec) in reviewers.iter().cloned().enumerate() {
6027                let wt = root.join(format!("review-{}", r + 1));
6028                // `clean -fdx` in a reset wipes setup's products, so every
6029                // round sets the seat up again.
6030                if wt.exists() {
6031                    git::reset_detached(&wt, &head).await?;
6032                } else {
6033                    git::worktree_add_detached(&repo, &wt, &head).await?;
6034                }
6035                self.setup_seat_worktree(&repo, &wt, false).await?;
6036                let seat_key = format!("review-{}", r + 1);
6037                // The seat starts the round on whoever answered it last, not
6038                // on the agent the spec names, so a failure is not re-paid.
6039                let spec = pick_start_spec(
6040                    &self.roles.reviewer_roster,
6041                    spec,
6042                    self.state.seat_history.get(&seat_key),
6043                );
6044                let seat = self.seat(&seat_key, &spec.id);
6045                jobs.push(SeatJob {
6046                    prompt: prompt::review(&prompt::ReviewCtx {
6047                        instruction: &self.state.instruction,
6048                        branch: &winner.branch,
6049                        base_short: &base_short,
6050                        stat: &stat,
6051                        patch: &patch,
6052                        verification: prev_verification.as_ref(),
6053                        prior: prior_section.as_deref(),
6054                        reviewers: reviewers.len(),
6055                        round,
6056                        rounds: max_rounds,
6057                        // A review-only run has no rankings, so nothing
6058                        // competed for this patch and the reviewer is told so.
6059                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6060                        lens: Lens::for_seat(r),
6061                        language: &language,
6062                        github_language: &self.state.config.graph.github_language,
6063                    }),
6064                    spec,
6065                    seat,
6066                    cwd: wt,
6067                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6068                    allow_write: false,
6069                    sessions,
6070                    artifacts: artifacts.clone(),
6071                    stem: format!("review-{round}-{}", r + 1),
6072                    handover: None,
6073                });
6074            }
6075
6076            drop(coordination);
6077            self.state.event(
6078                "review",
6079                format!(
6080                    "round {round}: {} reviewers on {}",
6081                    jobs.len(),
6082                    short(&head)
6083                ),
6084            );
6085            let mut quota_losses = Vec::new();
6086            let review_retries = self.state.config.graph.retries;
6087            let review_cache = self.state.config.cache_dir();
6088            let ctx = WaveCtx {
6089                carry_seats: true,
6090                pauses: Some((&self.pause, &self.interrupt)),
6091                run: &run_id,
6092                node: "review",
6093                prompts: &prompts,
6094                cache: review_cache.as_deref(),
6095                round: Some(round),
6096            };
6097            let results = ask_json_wave::<Review>(
6098                jobs,
6099                Arc::clone(&self.sem),
6100                review_retries,
6101                &self.roles.reviewer_roster,
6102                &ctx,
6103                &mut quota_losses,
6104                &mut self.state,
6105                &|_: &Review| Ok(()),
6106            )
6107            .await;
6108            if self.state.resource_block.is_some() {
6109                return self.stop_for_resource();
6110            }
6111            // Counted before the move below: how many of *this* round's
6112            // reviewer seats were lost to their own rate limit, as opposed to
6113            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6114            let round_quota_missing = quota_losses.len();
6115            self.state.quota.extend(quota_losses);
6116
6117            let mut records = Vec::new();
6118            let mut all_findings = Vec::new();
6119            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6120                let agent_id = seat.agent.clone();
6121                self.state.seats.insert(seat.key.clone(), seat);
6122                let mut record = ReviewRecord {
6123                    reviewer: r + 1,
6124                    agent: agent_id,
6125                    summary: String::new(),
6126                    findings: Vec::new(),
6127                    vote: None,
6128                    failed: None,
6129                    duration_ms: 0,
6130                    // Set for both outcomes: `failed: Some(_)` with
6131                    // `attempts > 0` is a seat every retry still lost, not a
6132                    // recovered one — only `failed: None` with `attempts > 0`
6133                    // reads as "answered after a nudge" (see this field's own
6134                    // doc).
6135                    attempts,
6136                };
6137                match res {
6138                    Ok((review, out)) => {
6139                        // Sanitized here, at the point every other piece of
6140                        // agent prose in this file is (candidate summaries,
6141                        // deliberation turns, vote reasons): a reviewer's own
6142                        // words are the one thing about it that could name
6143                        // it, and reconsideration below broadcasts this same
6144                        // summary and these same findings to every other
6145                        // seat on the panel.
6146                        record.summary =
6147                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6148                        record.vote = Some(review.vote);
6149                        record.duration_ms = out.duration_ms;
6150                        for (n, mut f) in review.findings.into_iter().enumerate() {
6151                            // ids are magi's, never the agent's: the fixer's
6152                            // adoption report is keyed by them.
6153                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6154                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6155                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6156                            // `file` is agent-supplied prose too, never
6157                            // checked against the real tree — the same
6158                            // exposure `title`/`detail` above have, just in
6159                            // a field easy to forget because it looks like a
6160                            // path rather than free text.
6161                            f.file = f
6162                                .file
6163                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6164                            all_findings.push(f.clone());
6165                            record.findings.push(f);
6166                        }
6167                        self.state.event(
6168                            "review",
6169                            format!(
6170                                "round {round}: reviewer {} voted {} with {} finding(s)",
6171                                r + 1,
6172                                review.vote.label(),
6173                                record.findings.len()
6174                            ),
6175                        );
6176                    }
6177                    Err(e) => {
6178                        record.failed = Some(e.to_string());
6179                        self.state.event(
6180                            "review",
6181                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6182                        );
6183                    }
6184                }
6185                records.push(record);
6186            }
6187
6188            // Tally the round's votes and, if they split, spend the one
6189            // round of reconsideration the split -> deliberate -> revote
6190            // shape `judge`/`vote` use for the panel, sized down to what a
6191            // read-only review round can afford: one round, and a revote
6192            // rather than an argument, because the panel already wrote its
6193            // reasoning down as findings the first time around.
6194            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6195            let vote_split =
6196                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6197            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6198            if vote_split {
6199                self.state.event(
6200                    "review",
6201                    format!(
6202                        "round {round}: votes split ({}) — one round of reconsideration",
6203                        initial_votes
6204                            .iter()
6205                            .map(|v| v.label())
6206                            .collect::<Vec<_>>()
6207                            .join(", ")
6208                    ),
6209                );
6210                // Seats read every seat's findings and votes, still numbered
6211                // and never named — the same anonymity `review` itself keeps.
6212                let panel: Vec<ReviewSeatReport<'_>> = records
6213                    .iter()
6214                    .filter_map(|r| {
6215                        r.vote.map(|vote| ReviewSeatReport {
6216                            reviewer: r.reviewer,
6217                            vote,
6218                            summary: &r.summary,
6219                            findings: &r.findings,
6220                        })
6221                    })
6222                    .collect();
6223
6224                let mut jobs = Vec::new();
6225                let mut seats_at = Vec::new();
6226                for (r, spec) in reviewers.iter().cloned().enumerate() {
6227                    // A seat with no initial vote has nothing to reconsider
6228                    // from and stays absent, the same as it stayed absent
6229                    // from `panel` above.
6230                    if records[r].vote.is_none() {
6231                        continue;
6232                    }
6233                    let wt = root.join(format!("review-{}", r + 1));
6234                    let seat_key = format!("review-{}", r + 1);
6235                    let spec = self.occupant(&seat_key, spec);
6236                    let seat = self.seat(&seat_key, &spec.id);
6237                    // A seat with no live session has already forgotten the
6238                    // initial review's prompt — restate the patch it is
6239                    // voting on, the same as `deliberate`/`vote` do for a
6240                    // judge in the same position.
6241                    // The panel already carries this seat's own review and
6242                    // vote, so restating the patch makes the prompt whole for
6243                    // a seat handed to another agent.
6244                    let build = |with_patch: bool| {
6245                        prompt::review_reconsider(&ReviewReconsiderCtx {
6246                            instruction: &self.state.instruction,
6247                            reviewer: r + 1,
6248                            lens: Lens::for_seat(r),
6249                            panel: &panel,
6250                            patch: with_patch.then_some(ReviewPatch {
6251                                branch: &winner.branch,
6252                                base_short: &base_short,
6253                                stat: &stat,
6254                                patch: &patch,
6255                            }),
6256                            round,
6257                            rounds: max_rounds,
6258                            language: &language,
6259                        })
6260                    };
6261                    let full = build(true);
6262                    let prompt = if has_context(&spec, &seat, sessions) {
6263                        build(false)
6264                    } else {
6265                        full.clone()
6266                    };
6267                    jobs.push(SeatJob {
6268                        prompt,
6269                        spec,
6270                        seat,
6271                        cwd: wt,
6272                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6273                        allow_write: false,
6274                        sessions,
6275                        artifacts: artifacts.clone(),
6276                        stem: format!("review-{round}-reconsider-{}", r + 1),
6277                        handover: Some(full),
6278                    });
6279                    seats_at.push(r);
6280                }
6281
6282                let mut recon_quota_losses = Vec::new();
6283                let recon_cache = self.state.config.cache_dir();
6284                let recon_ctx = WaveCtx {
6285                    carry_seats: true,
6286                    pauses: Some((&self.pause, &self.interrupt)),
6287                    run: &run_id,
6288                    node: "review",
6289                    prompts: &prompts,
6290                    cache: recon_cache.as_deref(),
6291                    round: Some(round),
6292                };
6293                let recon_results = ask_json_wave::<ReviewRevote>(
6294                    jobs,
6295                    Arc::clone(&self.sem),
6296                    review_retries,
6297                    &self.roles.reviewer_roster,
6298                    &recon_ctx,
6299                    &mut recon_quota_losses,
6300                    &mut self.state,
6301                    &|_: &ReviewRevote| Ok(()),
6302                )
6303                .await;
6304                if self.state.resource_block.is_some() {
6305                    return self.stop_for_resource();
6306                }
6307                self.state.quota.extend(recon_quota_losses);
6308
6309                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6310                    let agent_id = seat.agent.clone();
6311                    self.state.seats.insert(seat.key.clone(), seat);
6312                    let mut rec = ReviewRevoteRecord {
6313                        reviewer: r + 1,
6314                        agent: agent_id,
6315                        vote: None,
6316                        reason: String::new(),
6317                        failed: None,
6318                    };
6319                    match res {
6320                        Ok((rv, _)) => {
6321                            rec.vote = Some(rv.vote);
6322                            rec.reason =
6323                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6324                            self.state.event(
6325                                "review",
6326                                format!(
6327                                    "round {round}: reviewer {} revoted {}",
6328                                    r + 1,
6329                                    rv.vote.label()
6330                                ),
6331                            );
6332                        }
6333                        Err(e) => {
6334                            rec.failed = Some(e.to_string());
6335                            self.state.event(
6336                                "review",
6337                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6338                            );
6339                        }
6340                    }
6341                    reconsideration.push(rec);
6342                }
6343            } else if initial_votes.len() > 1 {
6344                self.state.event(
6345                    "review",
6346                    format!(
6347                        "round {round}: votes agreed ({}) — no reconsideration",
6348                        initial_votes[0].label()
6349                    ),
6350                );
6351            }
6352
6353            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6354            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6355            // A round that already has a blocking finding and a round left to
6356            // try is going back to the fixer no matter what `verify.e2e`
6357            // says, so running it first only spends the loop's slowest step
6358            // (minutes, for a Rust repo's full test suite) on a head about
6359            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6360            // runs once a round has no blocking findings left (see
6361            // `round_is_clean`, which a deferred — empty — `e2e` can never
6362            // satisfy since `blocking` is nonzero whenever this branch is
6363            // taken), and `stop_reviewing` forces a real run before it will
6364            // ever read a deferred round as green.
6365            let defer_e2e =
6366                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6367            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6368                let reason =
6369                    format!("{blocking} blocking finding(s) already required a fix this round");
6370                self.state.event(
6371                    "verify",
6372                    format!(
6373                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6374                         {}); it will run once a round has none left",
6375                        short(&head)
6376                    ),
6377                );
6378                (Vec::new(), false, true, Some(reason))
6379            } else {
6380                let e2e_commands = self.state.config.verify.e2e.clone();
6381                let cache_dir = self.state.config.cache_dir();
6382                let context = format!("round {round}");
6383                let wait = self.repository_wait();
6384                let (e2e, verify_retried) = with_cache_lease(
6385                    &mut self.state,
6386                    cache_dir.as_deref(),
6387                    "e2e",
6388                    "e2e",
6389                    &winner.worktree,
6390                    &head,
6391                    verify_timeout,
6392                    &context,
6393                    &wait,
6394                    |state, budget| {
6395                        let shell = shell.clone();
6396                        let e2e_commands = e2e_commands.clone();
6397                        let worktree = winner.worktree.clone();
6398                        let context = context.clone();
6399                        async move {
6400                            run_e2e_with_retry(
6401                                state,
6402                                &shell,
6403                                &e2e_commands,
6404                                &worktree,
6405                                budget,
6406                                &context,
6407                            )
6408                            .await
6409                        }
6410                    },
6411                )
6412                .await?;
6413                (e2e, verify_retried, false, None)
6414            };
6415
6416            let expected = records.len();
6417            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6418            let incomplete = answered < expected;
6419            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6420            let policy = self.state.config.graph.incomplete_review;
6421            let clean = round_is_clean(
6422                blocking,
6423                e2e_ok,
6424                answered,
6425                expected,
6426                round_quota_missing,
6427                policy,
6428            );
6429
6430            let mut round_record = ReviewRound {
6431                round,
6432                head: head.clone(),
6433                verified_head: None,
6434                verified_at: None,
6435                reviews: records,
6436                e2e,
6437                verify_retried,
6438                e2e_deferred,
6439                e2e_defer_reason,
6440                fix: None,
6441                blocking,
6442                answered,
6443                expected,
6444                clean,
6445                progressed: false,
6446                vote_split,
6447                reconsideration,
6448                verdict: None,
6449            };
6450            // The final vote per seat is its revote where reconsideration
6451            // ran and answered, its initial vote otherwise — the same
6452            // fallback `tally` uses for a judge whose private vote failed.
6453            round_record.verdict = ReviewVote::worst(
6454                round_record
6455                    .final_votes()
6456                    .into_iter()
6457                    .map(|(_, _, vote)| vote),
6458            );
6459            // Which commit and when magi actually attempted to check —
6460            // known the moment a command was dispatched against `head`,
6461            // whether or not it finished: a resource-blocked attempt still
6462            // targeted a specific commit at a specific time, and leaving
6463            // that unrecorded is exactly what made `verification_summary`
6464            // report a fresh attempt as "commit unknown ... recorded before
6465            // this was tracked", indistinguishable from a genuinely old,
6466            // untracked record. Only a deferred or unconfigured round never
6467            // ran at all and has nothing to record — see
6468            // `ReviewRound::verified_head`'s own doc.
6469            if !matches!(
6470                round_record.e2e_status(),
6471                E2eStatus::Deferred | E2eStatus::NotConfigured
6472            ) {
6473                round_record.verified_head = Some(head.clone());
6474                round_record.verified_at = Some(Timestamp::now());
6475            }
6476
6477            if incomplete {
6478                let missing: Vec<String> = round_record
6479                    .reviews
6480                    .iter()
6481                    .filter(|r| r.failed.is_some())
6482                    .map(|r| format!("review-{}", r.reviewer))
6483                    .collect();
6484                self.state.event(
6485                    "review",
6486                    format!(
6487                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6488                        missing.join(", ")
6489                    ),
6490                );
6491            }
6492
6493            if clean {
6494                self.state.event(
6495                    "review",
6496                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6497                        format!(
6498                            "round {round}: clean (warn policy, incomplete panel) — no \
6499                             blocking findings from the seats that answered, verification green"
6500                        )
6501                    } else if incomplete {
6502                        format!(
6503                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6504                             quorum) — no blocking findings from the seats that answered, \
6505                             verification green",
6506                            expected - answered
6507                        )
6508                    } else {
6509                        format!("round {round}: clean — no blocking findings, verification green")
6510                    },
6511                );
6512                self.state.reviews.push(round_record);
6513                self.state.status = RunStatus::Gating;
6514                self.state.save()?;
6515                return Ok(());
6516            }
6517
6518            // Nothing was raised and verification passed, but not every seat
6519            // answered and `round_is_clean` still refused to call it clean —
6520            // either a seat is missing for a reason other than its own quota
6521            // (a crash, a timeout, unparsable output — worth another try), or
6522            // every seat that could have answered lost its quota and nobody
6523            // is left to decide on: re-review rather than send the fixer
6524            // after a round with nothing to fix.
6525            if incomplete && blocking == 0 && e2e_ok {
6526                self.state.reviews.push(round_record);
6527                self.state.save()?;
6528                if round == max_rounds {
6529                    self.state.status = RunStatus::Blocked;
6530                    self.state.event(
6531                        "review",
6532                        format!(
6533                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6534                             refusing to call it clean",
6535                            expected - answered
6536                        ),
6537                    );
6538                    return Ok(());
6539                }
6540                continue;
6541            }
6542
6543            // Nothing for the fixer to act on (`blocking == 0`) and the only
6544            // reason this round is not clean is that magi itself never got
6545            // a command to run — the shared build cache, not the patch (see
6546            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6547            // the fixer would invite a change to appease contention that has
6548            // nothing to do with the diff, and would leave this attempt
6549            // sitting in the next round's prompt as if it were about an
6550            // earlier, superseded commit rather than what it actually is:
6551            // the same head, still waiting to be checked. Wait for it the
6552            // same way the final round's own contention is already handled,
6553            // whatever round this happens to be.
6554            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6555                self.state.reviews.push(round_record);
6556                return self
6557                    .stop_unresolved_review(
6558                        "the round's own verification could not run",
6559                        &shell,
6560                        &winner.worktree,
6561                    )
6562                    .await;
6563            }
6564
6565            if round == max_rounds {
6566                self.state.reviews.push(round_record);
6567                return self
6568                    .stop_reviewing(
6569                        &format!(
6570                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6571                        ),
6572                        &shell,
6573                        &winner.worktree,
6574                    )
6575                    .await;
6576            }
6577
6578            if !self.fix_review_round(&winner, round_record).await? {
6579                return Ok(());
6580            }
6581        }
6582        Ok(())
6583    }
6584
6585    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
6586    /// False means admission or continuation policy stopped the review loop.
6587    async fn fix_review_round(
6588        &mut self,
6589        winner: &Candidate,
6590        mut round_record: ReviewRound,
6591    ) -> Result<bool> {
6592        let round = round_record.round;
6593        let shell = self.state.config.shell();
6594        let step = self.run_fix_round(winner, &mut round_record).await?;
6595        if matches!(step, FixStep::Resource) {
6596            self.stop_for_resource()?;
6597            return Ok(false);
6598        }
6599        if self.state.reviews.len() < round {
6600            self.state.reviews.push(round_record);
6601        } else {
6602            self.state.reviews[round - 1] = round_record;
6603        }
6604        self.state.save()?;
6605
6606        // The fixer's own report never came back this round, even after
6607        // `continue_fix_report`'s own budget was spent on it — not an
6608        // ordinary "no report" (dropped stream, quota, plain failure),
6609        // which already reads that way and is left to the existing round
6610        // budget. Stopping here, rather than opening another round, is
6611        // what keeps a next reviewer/fixer wave from ever being
6612        // dispatched onto `winner.worktree` while whatever the seat's
6613        // last call may still have running there is unaccounted for: no
6614        // process liveness check exists (and none is being added — see
6615        // AGENTS.md/this task's own scope), so the only way to honour
6616        // "nothing starts before a valid report returns" is to not start
6617        // anything further on this worktree from this run at all.
6618        if matches!(step, FixStep::Recorded { report_lost: true }) {
6619            self.stop_reviewing(
6620                "the fixer's adoption report never came back, even after resuming its \
6621                         own seat; refusing to start another round against the same worktree \
6622                         while that is unresolved",
6623                &shell,
6624                &winner.worktree,
6625            )
6626            .await?;
6627            return Ok(false);
6628        }
6629
6630        let streak = self
6631            .state
6632            .reviews
6633            .iter()
6634            .rev()
6635            .take_while(|r| !r.progressed)
6636            .count();
6637        if streak >= STAGNANT_LIMIT {
6638            self.stop_reviewing(
6639                &format!("the tree has not moved against base for {streak} round(s) in a row"),
6640                &shell,
6641                &winner.worktree,
6642            )
6643            .await?;
6644            return Ok(false);
6645        }
6646        Ok(true)
6647    }
6648
6649    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
6650    async fn run_fix_round(
6651        &mut self,
6652        winner: &Candidate,
6653        round_record: &mut ReviewRound,
6654    ) -> Result<FixStep> {
6655        let round = round_record.round;
6656        let max_rounds = self.state.config.graph.review_rounds;
6657        let run_id = self.state.id.clone();
6658        let prompts = self.state.config.prompts.clone();
6659        let language = self.state.config.graph.language.clone();
6660        let sessions = self.state.config.graph.sessions;
6661        let artifacts = self.merge_repair_artifacts();
6662        let base = self.landing_base();
6663        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6664        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6665        let this_round_verification = round_record.verification_summary(&head);
6666        let all_findings: Vec<_> = round_record
6667            .reviews
6668            .iter()
6669            .flat_map(|r| r.findings.iter().cloned())
6670            .collect();
6671        // Fix. The winner's own implementer seat continues its conversation:
6672        // the competition is over, so context is pure benefit now.
6673        let blocking_findings: Vec<_> = all_findings
6674            .iter()
6675            .filter(|f| f.severity.blocks())
6676            .cloned()
6677            .collect();
6678        let fix_prompt = prompt::fix_in(
6679            &self.state.instruction,
6680            &blocking_findings,
6681            this_round_verification.as_ref(),
6682            round,
6683            max_rounds,
6684            &language,
6685            &self.state.config.graph.github_language,
6686        );
6687        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6688        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6689        let (job, seat, out) = self
6690            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6691                prompt: fix_prompt.clone(),
6692                spec,
6693                seat,
6694                cwd: winner.worktree.clone(),
6695                timeout,
6696                allow_write: true,
6697                sessions,
6698                artifacts: artifacts.clone(),
6699                stem: format!("fix-{round}"),
6700                handover: None,
6701            })
6702            .await;
6703        // Not asked: the round is not recorded (so a resume repeats it),
6704        // no fixer was handed over to, and the seat is left as it was.
6705        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6706            return Ok(FixStep::Resource);
6707        }
6708        let agent_id = seat.agent.clone();
6709
6710        let mut fix = FixRecord {
6711            agent: agent_id,
6712            addressed: Vec::new(),
6713            rejected: Vec::new(),
6714            notes: String::new(),
6715            committed: false,
6716            failed: None,
6717            duration_ms: 0,
6718            continuation: None,
6719        };
6720        let mut continuation = ContinuationRecord::not_needed();
6721        let mut final_seat = seat.clone();
6722        match out {
6723            AgentOutcome::Ok(o) => {
6724                fix.duration_ms = o.duration_ms;
6725                let parsed = verdict::extract_json::<FixReport>(&o.text);
6726                // A parsed report standing next to a command this same
6727                // reply's own CLI never confirmed the exit status of is
6728                // not a resolved answer — the identical `CommandEvidence`
6729                // `state.jobs` renders, read here instead of only on
6730                // display, per the completion judgment and the shown
6731                // record needing to agree.
6732                let incomplete_reason = match &parsed {
6733                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6734                        "the reply parsed, but it reported a command whose own CLI never \
6735                             confirmed an exit status"
6736                            .to_owned(),
6737                    ),
6738                    Ok(_) => None,
6739                    Err(e) => Some(e.to_string()),
6740                };
6741                match incomplete_reason {
6742                    None => {
6743                        let report = parsed.expect("checked Ok above");
6744                        fix.addressed = report.addressed;
6745                        fix.rejected = report.rejected;
6746                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6747                    }
6748                    Some(reason) => {
6749                        let (resumed_seat, resolved, failure, cont) = self
6750                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6751                            .await;
6752                        fix.duration_ms += cont.cumulative_wait_ms;
6753                        continuation = cont;
6754                        final_seat = resumed_seat;
6755                        match resolved {
6756                            Some(report) => {
6757                                fix.addressed = report.addressed;
6758                                fix.rejected = report.rejected;
6759                                fix.notes =
6760                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6761                            }
6762                            None => fix.failed = failure,
6763                        }
6764                    }
6765                }
6766            }
6767            // The CLI's raw error JSON is not a fix report to parse.
6768            AgentOutcome::Dropped(o) => {
6769                fix.duration_ms = o.duration_ms;
6770                let why = o
6771                    .dropped
6772                    .as_ref()
6773                    .map(|d| d.why.as_str())
6774                    .unwrap_or("the CLI ended the stream without delivering its answer");
6775                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6776            }
6777            AgentOutcome::Quota(o) => {
6778                self.state.quota.push(QuotaLoss {
6779                    seat: final_seat.key.clone(),
6780                    node: "fix".to_owned(),
6781                    at: Timestamp::now(),
6782                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6783                });
6784                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6785            }
6786            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6787        }
6788        fix.continuation = Some(continuation);
6789        self.state.seats.insert(final_seat.key.clone(), final_seat);
6790        let coordination = self.coordinate(&winner.worktree).await?;
6791        if let Ok(r) = git::rescue_commit(
6792            &winner.worktree,
6793            &format!("magi: review round {round} fixes (uncommitted work)"),
6794        )
6795        .await
6796        {
6797            self.state.note_withheld("fix", &r.withheld);
6798        }
6799        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6800        fix.committed = after != before;
6801        // Judged by what `git` says moved against base, never by the
6802        // fixer's own `addressed`/`rejected` count — see
6803        // `ReviewRound::progressed`. Propagated with `?`, the same as the
6804        // `patch` snapshot above: swallowing this error would default
6805        // `diff_after` to empty, which almost always differs from a
6806        // non-empty `patch` and reads as "progressed" — exactly backwards
6807        // for a `git` failure the stagnation check cannot see through.
6808        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6809        drop(coordination);
6810        let progressed = diff_after != patch;
6811        let commit_note = if fix.committed {
6812            "committed"
6813        } else {
6814            "NO new commit"
6815        };
6816        let tree_note = if progressed {
6817            "changed vs base"
6818        } else {
6819            "unchanged vs base"
6820        };
6821        self.state.event(
6822            "fix",
6823            match &fix.failed {
6824                // Distinct on purpose from "0 addressed, 0 rejected": the
6825                // fixer's own diff still landed (blocking counts do keep
6826                // falling round over round), only its adoption report did
6827                // not come back, so this must never read like every
6828                // finding was reviewed and declined.
6829                Some(reason) => {
6830                    format!(
6831                        "round {round}: fixer's adoption report was lost ({reason}); \
6832                             {commit_note}, tree {tree_note}"
6833                    )
6834                }
6835                None => format!(
6836                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6837                         {tree_note}{}",
6838                    fix.addressed.len(),
6839                    fix.rejected.len(),
6840                    if continuation.outcome == ContinuationOutcome::Resumed {
6841                        format!(
6842                            " (adoption report recovered after {} continuation(s))",
6843                            continuation.attempts
6844                        )
6845                    } else {
6846                        String::new()
6847                    },
6848                ),
6849            },
6850        );
6851        round_record.fix = Some(fix);
6852        round_record.progressed = progressed;
6853        Ok(FixStep::Recorded {
6854            report_lost: matches!(
6855                continuation.outcome,
6856                ContinuationOutcome::Exhausted
6857                    | ContinuationOutcome::QuotaLost
6858                    | ContinuationOutcome::NoSession
6859            ),
6860        })
6861    }
6862
6863    /// Decide, from the last recorded round's own verification, whether
6864    /// stopping the review loop is a hand-off or a genuine block.
6865    ///
6866    /// Called once the loop has given up trying — the round budget is spent,
6867    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
6868    /// findings still open, never while a round is still clean or the
6869    /// incomplete-panel case handled inline above. Gate and e2e are facts
6870    /// about the tree; a lingering review finding is an opinion, and this
6871    /// workload's own `magi stats` puts reviewer precision low enough
6872    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
6873    /// must not by itself stand between a green, verified change and the
6874    /// human who decides what to do with it. A red e2e is not an opinion, so
6875    /// that case still blocks, with the failing command and a tail of its
6876    /// output recorded here rather than left in `run.json` for someone to go
6877    /// find.
6878    ///
6879    /// A round that deferred its own e2e (see [`Config::graph`]'s
6880    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
6881    /// only because nothing ran, and treating an empty list as a passing one
6882    /// here is exactly the "deferred painted green" bug this function exists
6883    /// to not have. When the last round's own verification never resolved —
6884    /// deferred on purpose, or a real attempt the shared build cache blocked
6885    /// — this makes (or retries) the real run, on the actual worktree this
6886    /// loop is about to stop touching, before deciding anything. A
6887    /// resource-blocked attempt is likewise never read as either green or
6888    /// red: it is evidence about the machine, not the patch (see
6889    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
6890    /// blocked cache leaves this call without deciding rather than guessing
6891    /// — the caller retries on a later reentry.
6892    /// Record, once, that the review loop handed off over a blocking finding
6893    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
6894    /// `land` asks the owner even with `land_approval` off. Called from every
6895    /// path that concludes `Gating`; a reentry keeps the first record.
6896    fn record_contested_handoff(&mut self) {
6897        if self.state.contested_handoff.is_some() {
6898            return;
6899        }
6900        let Some(contested) = self
6901            .state
6902            .reviews
6903            .last()
6904            .and_then(ReviewRound::contested_handoff)
6905        else {
6906            return;
6907        };
6908        self.state.event(
6909            "review",
6910            format!(
6911                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6912                 wait for the owner's approval",
6913                contested.findings.len(),
6914                contested.rejecters.len()
6915            ),
6916        );
6917        self.state.contested_handoff = Some(contested);
6918    }
6919
6920    async fn stop_unresolved_review(
6921        &mut self,
6922        why: &str,
6923        shell: &[String],
6924        worktree: &Path,
6925    ) -> Result<()> {
6926        self.stop_reviewing_inner(why, shell, worktree, true).await
6927    }
6928
6929    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6930        self.stop_reviewing_inner(why, shell, worktree, false).await
6931    }
6932
6933    async fn stop_reviewing_inner(
6934        &mut self,
6935        why: &str,
6936        shell: &[String],
6937        worktree: &Path,
6938        repairable: bool,
6939    ) -> Result<()> {
6940        let round_idx = self.state.reviews.len() - 1;
6941        // A deferred round and a resource-blocked one are the same shape
6942        // here: neither has a real result yet, and both get one more
6943        // attempt. Read off `e2e_status` — the single source for this —
6944        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
6945        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
6946        // still retries instead of being read as a settled result the
6947        // instant it stops being empty.
6948        let needs_catchup_run = matches!(
6949            self.state.reviews[round_idx].e2e_status(),
6950            E2eStatus::Deferred | E2eStatus::ResourceBlocked
6951        );
6952        if needs_catchup_run {
6953            let round = self.state.reviews[round_idx].round;
6954            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6955            let commands = self.state.config.verify.e2e.clone();
6956            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6957            let cache_dir = self.state.config.cache_dir();
6958            let context = format!(
6959                "round {round}: verification unresolved, catching up before the final decision"
6960            );
6961            let wait = self.repository_wait();
6962            let (outcomes, verify_retried) = with_cache_lease(
6963                &mut self.state,
6964                cache_dir.as_deref(),
6965                "e2e",
6966                "e2e",
6967                worktree,
6968                &attempted_head,
6969                timeout,
6970                &context,
6971                &wait,
6972                |state, budget| {
6973                    let shell = shell.to_vec();
6974                    let commands = commands.clone();
6975                    let context = context.clone();
6976                    async move {
6977                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6978                            .await
6979                    }
6980                },
6981            )
6982            .await?;
6983            let last = &mut self.state.reviews[round_idx];
6984            last.e2e = outcomes;
6985            last.verify_retried = verify_retried;
6986            // Always the commit and time this attempt actually targeted,
6987            // whether or not it happens to equal the reviewed `head` and
6988            // whether or not a command finished — see
6989            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
6990            // attempt is recorded too, so a later reader sees "attempted
6991            // again at T2" rather than silence.
6992            last.verified_head = Some(attempted_head);
6993            last.verified_at = Some(Timestamp::now());
6994            if verify_inconclusive(&last.e2e) {
6995                // Still not a real result: `e2e_deferred` is left exactly
6996                // as it was, so `needs_catchup_run` above reads
6997                // `ResourceBlocked` (via `e2e_status`, which checks
6998                // `resource_blocked` before `e2e_deferred`) and retries
6999                // again on the next reentry, rather than recording
7000                // contention as a red e2e and blocking the run on it.
7001                self.state.save()?;
7002                return Ok(());
7003            }
7004            last.e2e_deferred = false;
7005        }
7006        let last = &self.state.reviews[round_idx];
7007        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7008
7009        match last.e2e_status() {
7010            E2eStatus::Failed => {
7011                let red: Vec<String> = last
7012                    .e2e
7013                    .iter()
7014                    .filter(|o| !o.ok())
7015                    .map(|o| {
7016                        format!(
7017                            "`{}` -> {:?}\n{}",
7018                            o.command,
7019                            o.code,
7020                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7021                        )
7022                    })
7023                    .collect();
7024                let can_repair = last.blocking == 0 && last.fix.is_none();
7025                self.state.reviews[round_idx].clean = false;
7026                self.state
7027                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7028                self.state.status = RunStatus::Blocked;
7029                if repairable
7030                    && can_repair
7031                    && self.state.reviews.len() < self.state.config.graph.review_rounds
7032                {
7033                    self.state.status = RunStatus::Reviewing;
7034                }
7035            }
7036            // `needs_catchup_run` above already retried once this call; if
7037            // it is still blocked, this is magi's own admission it could
7038            // not get a command to run, never a verdict on the patch — the
7039            // run is left exactly where a later reentry can retry again.
7040            E2eStatus::ResourceBlocked => {
7041                self.state.event(
7042                    "review",
7043                    format!(
7044                        "{why}; e2e could not run (shared build cache unavailable); not \
7045                         deciding yet"
7046                    ),
7047                );
7048            }
7049            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7050                self.state.event(
7051                    "review",
7052                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7053                );
7054                self.record_contested_handoff();
7055                self.state.status = RunStatus::Gating;
7056            }
7057        }
7058        self.state.save()?;
7059        Ok(())
7060    }
7061
7062    // ----------------------------------------------------------------- gate
7063
7064    async fn gate(&mut self) -> Result<()> {
7065        // Judged by the review record itself, not by `status`: a solo
7066        // candidate's `judge`/`deliberate` skip rewrites `status` on every
7067        // reentry (see `judge`), and trusting it here is exactly how a run
7068        // that exhausted its review budget got gated and merged a second
7069        // time around. `review_conclusion` recomputes the review loop's own
7070        // verdict from the round records themselves — `Gating` for a clean
7071        // round or a hand-off (see `stop_reviewing`), anything else means the
7072        // loop is still going or genuinely blocked.
7073        // A base the winner could not be replayed onto is a decision, not a
7074        // round: there is no landing tree to gate. Read as its own record for
7075        // the same reason the review verdict is.
7076        if self.state.status == RunStatus::Failed
7077            || self
7078                .state
7079                .base_sync
7080                .as_ref()
7081                .is_some_and(|s| s.conflict.is_some())
7082            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7083                != Some(RunStatus::Gating)
7084        {
7085            return Ok(());
7086        }
7087        if self.state.gate_ran {
7088            // `review_loop` derives its conclusion from the clean review
7089            // record on every reentry and therefore puts a completed run back
7090            // in `Gating`. A recorded gate is a stronger, terminal fact:
7091            // retain its original command output (or lack of any, for a repo
7092            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
7093            // doc) and restore `Blocked` on a real failure rather than
7094            // pretending the command is still running or running it a second
7095            // time. `gate_ran == false` remains the only shape — unattempted,
7096            // or a resource-blocked retry — that may still need to execute a
7097            // command.
7098            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7099                self.state.status = RunStatus::Blocked;
7100                self.state.save()?;
7101            }
7102            return Ok(());
7103        }
7104        let Some(winner) = self.state.winner().cloned() else {
7105            return Ok(());
7106        };
7107        self.state.status = RunStatus::Gating;
7108        let mut outcomes = self.run_gate(&winner).await?;
7109        loop {
7110            // A resource-blocked outcome means the gate command never actually
7111            // ran - the shared build cache could not be acquired or confirmed
7112            // fresh in time - which is evidence about the machine, not about
7113            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
7114            // Recording it as a red gate would mark a run `Blocked` on nothing
7115            // but contention magi has already logged; leaving `self.state.gate`
7116            // empty and `self.state.gate_ran` false instead keeps the shape
7117            // this function already treats as "still needs to run" (see the
7118            // early-return above), so the next call retries the command
7119            // rather than concluding anything.
7120            if verify_inconclusive(&outcomes) {
7121                self.state.save()?;
7122                return Ok(());
7123            }
7124            if outcomes.iter().all(CommandOutcome::ok) {
7125                break;
7126            }
7127            match self.gate_fix_round(&winner, &outcomes).await? {
7128                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7129                GateFix::Stop => break,
7130                GateFix::Defer => {
7131                    self.state.save()?;
7132                    return Ok(());
7133                }
7134            }
7135        }
7136        let passed = outcomes.iter().all(CommandOutcome::ok);
7137        self.state.gate = outcomes;
7138        self.state.gate_ran = true;
7139        self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7140        if !passed {
7141            self.state.status = RunStatus::Blocked;
7142            let spent = self.state.gate_fixes.len();
7143            self.state.event(
7144                "gate",
7145                if spent == 0 {
7146                    "gate failed; not merging".to_owned()
7147                } else {
7148                    format!("gate failed after {spent} gate-fix round(s); not merging")
7149                },
7150            );
7151        }
7152        self.state.save()?;
7153        Ok(())
7154    }
7155
7156    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7157    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7158    /// after review is clean and never on a candidate awaiting judging.
7159    ///
7160    /// Never fails the run: a non-zero exit or timeout is a warning and a
7161    /// recorded outcome, and the gate remains the single arbiter. Nothing
7162    /// configured means nothing happens - no event, no commit. `commit_all`
7163    /// commits any leftover change under the neutral identity and returns
7164    /// `false` when the tree is clean, so no empty commit is ever made.
7165    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7166        let commands = state.config.verify.pre_gate.clone();
7167        if commands.is_empty() {
7168            return;
7169        }
7170        let shell = state.config.shell();
7171        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7172        let (outcomes, _) = run_commands(
7173            state,
7174            "pre_gate",
7175            "pre_gate",
7176            0,
7177            &shell,
7178            &commands,
7179            &winner.worktree,
7180            timeout,
7181        )
7182        .await;
7183        for o in &outcomes {
7184            if !o.ok() {
7185                tracing::warn!(
7186                    "pre_gate `{}` failed ({:?}); the gate decides",
7187                    o.command,
7188                    o.code
7189                );
7190            }
7191            state.event(
7192                "pre_gate",
7193                format!(
7194                    "`{}` -> {}",
7195                    o.command,
7196                    if o.ok() {
7197                        "pass".to_owned()
7198                    } else {
7199                        format!(
7200                            "FAIL ({:?})\n{}",
7201                            o.code,
7202                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7203                        )
7204                    }
7205                ),
7206            );
7207        }
7208        state.pre_gate = outcomes;
7209        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7210            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7211                Ok(head) => {
7212                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7213                    state.pre_gate_commit = Some(head);
7214                }
7215                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7216            },
7217            Ok(false) => {}
7218            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7219        }
7220        if let Err(e) = state.save() {
7221            tracing::warn!("could not persist the pre_gate record: {e:#}");
7222        }
7223    }
7224
7225    /// Run `verify.gate` once against the winner's current tree, logging one
7226    /// event per command. Empty when nothing is configured.
7227    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7228        let wait = self.repository_wait();
7229        run_landing_gate(&mut self.state, winner, &wait).await
7230    }
7231
7232    /// One bounded fix round for a failing gate.
7233    ///
7234    /// The fixer is told the failure came from the gate itself, not from a
7235    /// reviewer, and is shown the failed commands, their exit codes and a tail
7236    /// of their output - whatever `[verify].gate` holds, nothing here knows
7237    /// what those commands run. Only a normal non-zero exit that printed
7238    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7239    /// command or a full disk says nothing about the code, and a fixer sent
7240    /// after it can only appease the machine. The round is judged by what git
7241    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7242    /// again before the gate does, so a fix cannot trade a green gate for a
7243    /// red e2e unnoticed.
7244    async fn gate_fix_round(
7245        &mut self,
7246        winner: &Candidate,
7247        outcomes: &[CommandOutcome],
7248    ) -> Result<GateFix> {
7249        let cap = self.state.config.graph.gate_fix_rounds;
7250        let spent = self.state.gate_fixes.len();
7251        if spent >= cap {
7252            if cap > 0 {
7253                self.state.event(
7254                    "gate",
7255                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7256                );
7257            }
7258            return Ok(GateFix::Stop);
7259        }
7260        if !gate_fixable(outcomes) {
7261            self.state.event(
7262                "gate",
7263                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7264                 command or similar); not spending a fix round on it",
7265            );
7266            return Ok(GateFix::Stop);
7267        }
7268        let min_free = self.state.config.disk.min_free_bytes;
7269        if min_free > 0 {
7270            match crate::disk::free_bytes(&winner.worktree) {
7271                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7272                Ok(free) => {
7273                    self.state.event(
7274                        "gate",
7275                        format!(
7276                            "only {free} bytes free ({min_free} required by `[disk] \
7277                             min_free_bytes`); not spending a fix round on a failure the disk \
7278                             may explain"
7279                        ),
7280                    );
7281                    return Ok(GateFix::Stop);
7282                }
7283                Err(e) => {
7284                    self.state.event(
7285                        "gate",
7286                        format!("free disk space could not be measured ({e:#}); no fix round"),
7287                    );
7288                    return Ok(GateFix::Stop);
7289                }
7290            }
7291        }
7292
7293        let attempt = spent + 1;
7294        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7295        let base = self.landing_base();
7296        let fix_prompt = prompt::gate_fix_in(
7297            &self.state.instruction,
7298            &failed,
7299            attempt,
7300            cap,
7301            &self.state.config.graph.language,
7302            &self.state.config.graph.github_language,
7303        );
7304        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7305        let sessions = self.state.config.graph.sessions;
7306        let artifacts = self.merge_repair_artifacts();
7307        self.state.event(
7308            "gate",
7309            format!("gate failed; gate-fix round {attempt} of {cap}"),
7310        );
7311        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7312        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7313        // Persist before invoking the fixer: it may commit and then be
7314        // interrupted before its result (or the following e2e) is saved.
7315        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7316            head: before.clone(),
7317            passed: false,
7318            outcomes: Vec::new(),
7319        });
7320        self.state.save()?;
7321        let (_, seat, out) = self
7322            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7323                prompt: fix_prompt.clone(),
7324                spec,
7325                seat,
7326                cwd: winner.worktree.clone(),
7327                timeout,
7328                allow_write: true,
7329                sessions,
7330                artifacts: artifacts.clone(),
7331                stem: format!("gate-fix-{attempt}"),
7332                handover: None,
7333            })
7334            .await;
7335        // The fixer was never asked: no round is spent (nothing is recorded)
7336        // and a later reentry asks the same agent again.
7337        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7338            return Ok(GateFix::Defer);
7339        }
7340        let mut record = GateFixRecord {
7341            agent: seat.agent.clone(),
7342            failed,
7343            notes: String::new(),
7344            committed: false,
7345            error: None,
7346        };
7347        match out {
7348            AgentOutcome::Ok(o) => {
7349                // A missing report is not a failed fix: the round is judged
7350                // by the tree below, and the report only carries prose.
7351                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7352                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7353                }
7354            }
7355            AgentOutcome::Dropped(_) => {
7356                record.error = Some("the CLI dropped the stream".to_owned());
7357            }
7358            AgentOutcome::Quota(o) => {
7359                self.state.quota.push(QuotaLoss {
7360                    seat: seat.key.clone(),
7361                    node: "gate-fix".to_owned(),
7362                    at: Timestamp::now(),
7363                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7364                });
7365                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7366            }
7367            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7368        }
7369        self.state.seats.insert(seat.key.clone(), seat);
7370        let coordination = self.coordinate(&winner.worktree).await?;
7371        if let Ok(r) = git::rescue_commit(
7372            &winner.worktree,
7373            &format!("magi: gate fix {attempt} (uncommitted work)"),
7374        )
7375        .await
7376        {
7377            self.state.note_withheld("gate-fix", &r.withheld);
7378        }
7379        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7380        record.committed = after != before;
7381        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7382        let note = record.error.clone();
7383        self.state.gate_fixes.push(record);
7384        self.state.save()?;
7385        if !changed {
7386            self.state.event(
7387                "gate",
7388                match note {
7389                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7390                    None => format!("gate-fix round {attempt}: the tree did not change"),
7391                },
7392            );
7393            return Ok(GateFix::Stop);
7394        }
7395        self.state.event(
7396            "gate",
7397            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7398        );
7399
7400        drop(coordination);
7401        let wait = self.repository_wait();
7402        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7403    }
7404
7405    // ---------------------------------------------------------------- merge
7406
7407    /// One read-only rewrite by the summary's author; a text the gate still
7408    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7409    /// applies on their word or on silence. `None` means the run is parked on
7410    /// that question and nothing may be pushed or opened yet.
7411    async fn guarded_pr_message(
7412        &mut self,
7413        winner: &Candidate,
7414        facts: Option<&BranchFacts>,
7415        posting: bool,
7416    ) -> Result<Option<PrMessage>> {
7417        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7418        let head = match final_head.as_deref() {
7419            Some(h) => crate::pr_outcome::FinalHead::At(h),
7420            None => crate::pr_outcome::FinalHead::Unknown,
7421        };
7422        // Only a pull request that is about to be posted pays for a summary.
7423        let summarized = if posting {
7424            Some(self.summarize_change(winner, final_head.as_deref()).await)
7425        } else {
7426            None
7427        };
7428        // A busy shared resource is not a permanent cause: publishing an
7429        // "unavailable" description would stick (a later head move never
7430        // edits the body). Stop before anything is pushed or opened; the
7431        // run is `Blocked` and a resume asks the summary again.
7432        if self.state.resource_block.is_some() {
7433            self.state.event(
7434                "pr-summary",
7435                "a shared resource was busy; nothing was published, resume to summarize again",
7436            );
7437            return Ok(None);
7438        }
7439        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7440            anyhow::bail!(
7441                "the final commit moved during public description generation; retry before publishing"
7442            );
7443        }
7444        let note = match (&summarized, &self.state.change_summary) {
7445            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7446            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7447            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7448            (None, _) => ChangeNote::NotAsked,
7449        };
7450        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7451        if !posting {
7452            let identity = crate::scrub::Identity::current();
7453            return Ok(Some(PrMessage {
7454                title: crate::scrub::scrub(&pr.title, &identity),
7455                body: crate::scrub::scrub(&pr.body, &identity),
7456            }));
7457        }
7458        // This very text was already rejected and asked about: no second
7459        // judge call, rewrite or question, just carry the record on.
7460        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7461        if let Some(known) = self
7462            .state
7463            .github_text
7464            .clone()
7465            .filter(|g| g.fingerprint == raw_fp)
7466        {
7467            return self.settle_withheld(winner, &pr, known).await;
7468        }
7469        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7470        let mut violations =
7471            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7472        let identity = crate::scrub::Identity::current();
7473        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7474            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7475            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7476        {
7477            violations.push(crate::github_text::Violation::SensitiveData);
7478        }
7479        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7480        // only a language violation starts one; sensitive data alone goes
7481        // straight to the owner, with the redacted text as the fallback.
7482        let has_language = violations
7483            .iter()
7484            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7485        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7486            self.state.event(
7487                "github-text",
7488                if has_language {
7489                    format!("description rejected: {violations:?}; requesting one rewrite")
7490                } else {
7491                    format!("description rejected: {violations:?}; asking the owner")
7492                },
7493            );
7494            let mut rewritten = false;
7495            let mut rejected_rewrite = None;
7496            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7497            {
7498                let output = text;
7499                if let Some(title) = summary_title(&output) {
7500                    // Judge the redacted candidate that will actually be posted.
7501                    // Sensitive data alone must not discard a useful rewrite.
7502                    let raw_body = summary_without_title(&output);
7503                    let clean_title = crate::scrub::scrub(&title, &identity);
7504                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7505                    let redacted = clean_title != title || body != raw_body;
7506                    let title = clean_title;
7507                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7508                    if !complete {
7509                        self.state.event(
7510                            "github-text",
7511                            "description rewrite omitted required findings",
7512                        );
7513                    }
7514                    if !body.trim().is_empty() {
7515                        body.push_str("\n\n");
7516                        body.push_str(&public_pr_evidence(&self.state, head));
7517                    }
7518                    let marker = format!("magi:run/{}", self.state.id);
7519                    if !body.trim().is_empty() && !body.contains(&marker) {
7520                        body.push_str(&format!("\n\n{marker}"));
7521                    }
7522                    let rewrite = PrMessage {
7523                        title: title.clone(),
7524                        body: body.clone(),
7525                    };
7526                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7527                    let mut failed = crate::github_text::check_for(
7528                        &self.state.config.graph,
7529                        &title,
7530                        &body,
7531                        redecision,
7532                    );
7533                    if (!crate::github_text::shareable(&title)
7534                        || !crate::github_text::shareable(&body))
7535                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7536                    {
7537                        failed.push(crate::github_text::Violation::SensitiveData);
7538                    }
7539                    if complete
7540                        && !title.trim().is_empty()
7541                        && !body.trim().is_empty()
7542                        && !failed.is_empty()
7543                    {
7544                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7545                            title: title.clone(),
7546                            body: body.clone(),
7547                            failed: failed
7548                                .iter()
7549                                .map(|v| crate::github_text::category(*v).to_owned())
7550                                .collect(),
7551                        });
7552                    }
7553                    if complete
7554                        && !title.trim().is_empty()
7555                        && !raw_body.trim().is_empty()
7556                        && failed.is_empty()
7557                    {
7558                        decision = redecision;
7559                        pr = PrMessage { title, body };
7560                        rewritten = true;
7561                        self.state
7562                            .event("github-text", "description rewrite passed");
7563                        if redacted {
7564                            self.state.event(
7565                                "github-text",
7566                                "sensitive data removed from description rewrite",
7567                            );
7568                        }
7569                    }
7570                }
7571            }
7572            if !rewritten {
7573                self.state.event(
7574                    "github-text",
7575                    "description rewrite unavailable or rejected; asking the owner",
7576                );
7577                if let Some(w) =
7578                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7579                {
7580                    // A different text than the one asked about before: that
7581                    // question no longer describes anything.
7582                    self.retire_github_text_question("the withheld text changed");
7583                    let record = crate::run::GithubTextAsk {
7584                        rewrite: rejected_rewrite,
7585                        fingerprint: raw_fp,
7586                        title: w.title,
7587                        body: w.body,
7588                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7589                        question: None,
7590                        asks: 0,
7591                        resolved: false,
7592                        chosen_title: None,
7593                        choice: None,
7594                    };
7595                    return self.settle_withheld(winner, &pr, record).await;
7596                }
7597            }
7598        }
7599        let (title, body) =
7600            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7601        Ok(Some(PrMessage { title, body }))
7602    }
7603
7604    /// Describe the whole change from its final diff in either run mode.
7605    ///
7606    /// The summary is tied to the final commit and to the merge base of the
7607    /// resolved remote base; a record for any other pair is dropped and the
7608    /// agent asked again, so a description never outlives the diff it was
7609    /// written from. The seat is read-only and the prompt tells it to read the
7610    /// diff and the code around it. `[roles] synthesizer` selects the agents
7611    /// (unset: the default pick), each asked at most once, advancing on an
7612    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7613    /// carries the reason it is unavailable; nothing is invented then.
7614    async fn summarize_change(
7615        &mut self,
7616        winner: &Candidate,
7617        head: Option<&str>,
7618    ) -> std::result::Result<(), String> {
7619        let Some(head) = head else {
7620            return Err("the final commit could not be read".to_owned());
7621        };
7622        let remote = self.state.config.merge.remote.clone();
7623        let tracking = format!("{remote}/{}", self.state.base_branch);
7624        if !git::rev_exists(&winner.worktree, &tracking).await {
7625            return Err(format!("the remote base `{tracking}` could not be read"));
7626        }
7627        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7628            Ok(b) if !b.is_empty() => b,
7629            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7630        };
7631        let evidence = crate::pr_outcome::summary_evidence(
7632            &self.state,
7633            crate::pr_outcome::FinalHead::At(head),
7634        );
7635        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7636        if self
7637            .state
7638            .change_summary
7639            .as_ref()
7640            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7641        {
7642            return Ok(());
7643        }
7644        if self.state.change_summary.take().is_some() {
7645            self.state.event(
7646                "pr-summary",
7647                "the final diff or evidence changed; the earlier change summary was discarded",
7648            );
7649        }
7650        // Save the source snapshot even if no generation agent is available.
7651        if let Err(e) = write_artifact(
7652            &self.state,
7653            &format!("public-description-{fingerprint}.md"),
7654            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7655        ) {
7656            tracing::warn!("could not save public description evidence: {e:#}");
7657        }
7658        let chain = match agent::pick_chain(
7659            &self.state.config.agents,
7660            self.state.config.roles.synthesizer.as_ref(),
7661            &agent::installed,
7662            "synthesizer",
7663        ) {
7664            Ok(chain) => chain,
7665            Err(e) => {
7666                tracing::warn!("no agent for the change summary: {e:#}");
7667                self.state.event("pr-summary", "no summary agent available");
7668                return Err("no agent was available to summarize it".to_owned());
7669            }
7670        };
7671        let subjects = git::subjects(&winner.worktree, &base, head)
7672            .await
7673            .unwrap_or_default();
7674        let stat = git::diff_stat(&winner.worktree, &base, head)
7675            .await
7676            .unwrap_or_default();
7677        let stat: String = stat.chars().take(6_000).collect();
7678        let mut prompt = crate::prompt::change_summary_in(
7679            &base,
7680            head,
7681            &subjects,
7682            stat.trim(),
7683            &self.state.config.graph.github_language,
7684        );
7685        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7686        let sessions = self.state.config.graph.sessions;
7687        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7688        let prompts = self.state.config.prompts.clone();
7689        let cache = self.state.config.cache_dir();
7690        let run = self.state.id.clone();
7691        for (i, spec) in chain.into_iter().enumerate() {
7692            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7693            let github_spec = match agent::for_github_text(
7694                &spec,
7695                &self.state.config.graph.github_language,
7696                &winner.worktree,
7697            ) {
7698                Ok(spec) => spec,
7699                Err(e) => {
7700                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7701                    continue;
7702                }
7703            };
7704            let job = SeatJob {
7705                spec: github_spec,
7706                seat,
7707                cwd: winner.worktree.clone(),
7708                prompt: prompt.clone(),
7709                timeout,
7710                allow_write: false,
7711                sessions,
7712                artifacts: agent::artifacts_dir(&self.state.dir()),
7713                stem: if i == 0 {
7714                    "change-summary".to_owned()
7715                } else {
7716                    format!("change-summary-{}", spec.id)
7717                },
7718                handover: None,
7719            };
7720            let ctx = WaveCtx {
7721                run: &run,
7722                node: "pr-summary",
7723                prompts: &prompts,
7724                cache: cache.as_deref(),
7725                round: None,
7726                carry_seats: false,
7727                pauses: Some((&self.pause, &self.interrupt)),
7728            };
7729            let (seat, outcome) =
7730                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7731            self.state.seats.insert(seat.key.clone(), seat);
7732            let text = match outcome {
7733                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7734                AgentOutcome::ResourceBlocked(why) => {
7735                    // No agent was asked: the run stops resumably before
7736                    // anything is published (see `guarded_pr_message`).
7737                    if self.state.resource_block.is_none() {
7738                        self.state.resource_block = Some(crate::run::ResourceBlock {
7739                            node: "pr-summary".to_owned(),
7740                            kind: "shared resource".to_owned(),
7741                            detail: why,
7742                            at: Timestamp::now(),
7743                        });
7744                    }
7745                    return Err("a shared resource was busy".to_owned());
7746                }
7747                _ => continue,
7748            };
7749            let blind = &self.state.config.blind;
7750            let Some(title) = summary_title(&text) else {
7751                continue;
7752            };
7753            let body: String = summary_without_title(&text)
7754                .lines()
7755                .filter(|l| !l.contains("magi:run/"))
7756                .collect::<Vec<_>>()
7757                .join("\n");
7758            let body = blind::sanitize_prose(body.trim(), blind);
7759            let title = blind::sanitize_prose(&title, blind);
7760            if body.trim().is_empty() || title.trim().is_empty() {
7761                continue;
7762            }
7763            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7764                continue;
7765            }
7766            // The branch must still be the one the agent was shown.
7767            if git::rev_parse(&winner.worktree, "HEAD")
7768                .await
7769                .ok()
7770                .as_deref()
7771                != Some(head)
7772            {
7773                return Err("the branch moved while the summary was written".to_owned());
7774            }
7775            if git::merge_base(&winner.worktree, &tracking, head)
7776                .await
7777                .ok()
7778                .as_deref()
7779                != Some(base.as_str())
7780                || crate::pr_outcome::summary_evidence(
7781                    &self.state,
7782                    crate::pr_outcome::FinalHead::At(head),
7783                ) != evidence
7784            {
7785                return Err("the summary evidence changed while it was written".to_owned());
7786            }
7787            self.state.change_summary = Some(crate::run::ChangeSummary {
7788                head: head.to_owned(),
7789                base,
7790                evidence: fingerprint,
7791                title,
7792                body,
7793            });
7794            self.state.event(
7795                "pr-summary",
7796                format!("change summary written by {} for the final diff", spec.id),
7797            );
7798            if let Err(e) = self.state.save() {
7799                tracing::warn!("could not save the change summary: {e:#}");
7800            }
7801            return Ok(());
7802        }
7803        self.state
7804            .event("pr-summary", "no agent produced a change summary");
7805        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7806    }
7807
7808    /// One read-only rewrite request for a withheld description. The
7809    /// implementer's own conversation is tried first when it exists and can be
7810    /// resumed; then the `[roles] language_judge` chain (or the default pick)
7811    /// writes it in a separate conversation, skipping the implementer's agent.
7812    /// The prompt carries the whole text, so no agent needs any memory. Each
7813    /// agent is asked at most once and the chain advances on an error, quota,
7814    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
7815    /// that has run out of quota must not end the rewrite. Nothing is recorded
7816    /// as a handover.
7817    async fn rewrite_pr_text(
7818        &mut self,
7819        winner: &Candidate,
7820        pr: &PrMessage,
7821        violations: &[crate::github_text::Violation],
7822    ) -> Option<String> {
7823        let prompt = format!(
7824            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7825            pr.title, pr.body
7826        );
7827        let sessions = self.state.config.graph.sessions;
7828        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7829        // (spec, the resumed implementer's seat). A fallback seat is created
7830        // only when its turn comes, so an unused fallback leaves no record.
7831        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
7832        let key = format!("impl-{}", winner.label);
7833        if let Some(spec) = self
7834            .state
7835            .config
7836            .agents
7837            .iter()
7838            .find(|a| a.id == winner.agent)
7839            .cloned()
7840            && let Some(existing) = self.state.seats.get(&key)
7841            && existing.agent == spec.id
7842            && agent::has_session(spec.kind, existing, sessions)
7843        {
7844            attempts.push((spec, Some(existing.clone())));
7845        }
7846        match agent::pick_chain(
7847            &self.state.config.agents,
7848            self.state.config.roles.language_judge.as_ref(),
7849            &agent::installed,
7850            "language_judge",
7851        ) {
7852            Ok(chain) => {
7853                for spec in chain {
7854                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
7855                        continue;
7856                    }
7857                    attempts.push((spec, None));
7858                }
7859            }
7860            Err(e) if attempts.is_empty() => {
7861                tracing::warn!("no agent for the description rewrite: {e:#}");
7862                self.state
7863                    .event("github-text", "no rewrite agent available");
7864                return None;
7865            }
7866            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
7867        }
7868        let prompts = self.state.config.prompts.clone();
7869        let cache = self.state.config.cache_dir();
7870        let run = self.state.id.clone();
7871        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
7872            let resumed = resumed_seat.is_some();
7873            let seat = resumed_seat.unwrap_or_else(|| {
7874                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
7875            });
7876            let github_spec = match agent::for_github_text(
7877                &spec,
7878                &self.state.config.graph.github_language,
7879                &winner.worktree,
7880            ) {
7881                Ok(spec) => spec,
7882                Err(e) => {
7883                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7884                    continue;
7885                }
7886            };
7887            let job = SeatJob {
7888                spec: github_spec,
7889                seat,
7890                cwd: winner.worktree.clone(),
7891                prompt: prompt.clone(),
7892                timeout: retry_budget(timeout_full, resumed),
7893                allow_write: false,
7894                sessions,
7895                artifacts: agent::artifacts_dir(&self.state.dir()),
7896                stem: if i == 0 {
7897                    "github-text-rewrite".to_owned()
7898                } else {
7899                    format!("github-text-rewrite-{}", spec.id)
7900                },
7901                handover: None,
7902            };
7903            let ctx = WaveCtx {
7904                run: &run,
7905                node: "github-text",
7906                prompts: &prompts,
7907                cache: cache.as_deref(),
7908                round: None,
7909                carry_seats: false,
7910                pauses: Some((&self.pause, &self.interrupt)),
7911            };
7912            let (seat, outcome) =
7913                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7914            self.state.seats.insert(seat.key.clone(), seat);
7915            match outcome {
7916                AgentOutcome::Ok(o)
7917                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7918                {
7919                    return Some(o.text);
7920                }
7921                AgentOutcome::ResourceBlocked(_) => return None,
7922                _ => {}
7923            }
7924        }
7925        None
7926    }
7927
7928    /// Abandon the standing posting-gate question, if any.
7929    fn retire_github_text_question(&mut self, why: &str) {
7930        let Some(id) = self
7931            .state
7932            .github_text
7933            .as_ref()
7934            .and_then(|g| g.question.clone())
7935        else {
7936            return;
7937        };
7938        let _ = ask::Questions::open().update(&id, |q| {
7939            q.abandon(why);
7940            Ok(())
7941        });
7942    }
7943
7944    /// Carry a withheld text to its end: file the question, park while it is
7945    /// open, and on an answer, silence or timeout pick the text to post. The
7946    /// decision is saved before it is used, so it applies once.
7947    async fn settle_withheld(
7948        &mut self,
7949        winner: &Candidate,
7950        pr: &PrMessage,
7951        mut g: crate::run::GithubTextAsk,
7952    ) -> Result<Option<PrMessage>> {
7953        use crate::github_text as gt;
7954        let store = ask::Questions::open();
7955        let timeout = self.state.config.graph.answer_timeout;
7956        if !g.resolved {
7957            let mut standing = g
7958                .question
7959                .as_ref()
7960                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7961            if let Some(q) = standing.as_ref()
7962                && q.status.open()
7963                && gt::expired(q, timeout)
7964            {
7965                let why = format!("no answer within {}s of asking", timeout.max(1));
7966                standing = store
7967                    .update(&q.id, |q| {
7968                        q.abandon(&why);
7969                        Ok(())
7970                    })
7971                    .ok()
7972                    .map(|(q, ())| q);
7973            }
7974            let mut ask_again: Option<Option<Vec<String>>> = None;
7975            match standing {
7976                // Recorded but never written (a stop between the two saves),
7977                // or removed: ask again without spending the retry.
7978                None if g.asks <= 1 => {
7979                    g.asks = 0;
7980                    ask_again = Some(None);
7981                }
7982                None => {
7983                    g.resolved = true;
7984                    g.chosen_title = None;
7985                    g.choice = Some(GithubTextChoice::Fallback);
7986                }
7987                Some(q) if q.status.open() => {
7988                    self.park_on_github_text(&g)?;
7989                    return Ok(None);
7990                }
7991                Some(q) => match gt::read_reply(&q) {
7992                    gt::Reply::Fallback => {
7993                        g.resolved = true;
7994                        g.chosen_title = None;
7995                        g.choice = Some(GithubTextChoice::Fallback);
7996                    }
7997                    gt::Reply::Original => {
7998                        g.resolved = true;
7999                        g.chosen_title = None;
8000                        g.choice = Some(GithubTextChoice::Original);
8001                    }
8002                    gt::Reply::Rewrite => {
8003                        g.resolved = true;
8004                        g.chosen_title = None;
8005                        g.choice = Some(if g.rewrite.is_some() {
8006                            GithubTextChoice::Rewrite
8007                        } else {
8008                            GithubTextChoice::Fallback
8009                        });
8010                    }
8011                    gt::Reply::Title(text) => {
8012                        let first = text
8013                            .lines()
8014                            .map(str::trim)
8015                            .find(|l| !l.is_empty())
8016                            .unwrap_or("");
8017                        let decision = if first.is_empty() {
8018                            None
8019                        } else {
8020                            let candidate = PrMessage {
8021                                title: first.to_owned(),
8022                                body: String::new(),
8023                            };
8024                            self.judge_pr_language(&winner.worktree, &candidate).await
8025                        };
8026                        match gt::vet_title(
8027                            &text,
8028                            decision,
8029                            self.state.config.graph.github_english(),
8030                        ) {
8031                            Ok(title) => {
8032                                g.resolved = true;
8033                                g.chosen_title = Some(title);
8034                                g.choice = Some(GithubTextChoice::MyTitle);
8035                            }
8036                            Err(cats) if g.asks < 2 => {
8037                                ask_again = Some(Some(
8038                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8039                                ))
8040                            }
8041                            Err(_) => {
8042                                g.resolved = true;
8043                                g.chosen_title = None;
8044                                g.choice = Some(GithubTextChoice::Fallback);
8045                            }
8046                        }
8047                    }
8048                },
8049            }
8050            if let Some(rejected) = ask_again {
8051                // A retry (after a rejected replacement title) is still about
8052                // the original text: the same ORIGINAL / FALLBACK comparison.
8053                let w = gt::Withheld {
8054                    title: g.title,
8055                    body: g.body,
8056                    categories: g.categories.clone(),
8057                };
8058                // Named by the original text's fingerprint alone, so a resume
8059                // rewrites the same file and the detail stays identical.
8060                let artifact = match crate::run::write_artifact(
8061                    &self.state,
8062                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
8063                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8064                ) {
8065                    Ok(p) => Some(p),
8066                    Err(e) => {
8067                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
8068                        None
8069                    }
8070                };
8071                let (detail, comparison) = gt::question_content(
8072                    &w,
8073                    &pr.title,
8074                    &pr.body,
8075                    &self.state.id,
8076                    rejected.as_deref(),
8077                    artifact.as_deref(),
8078                    g.rewrite.as_ref(),
8079                );
8080                let mut q = ask::Question::new(
8081                    self.state.id.clone(),
8082                    gt::ASK_NODE.to_owned(),
8083                    gt::ASK_SEAT.to_owned(),
8084                    gt::question_summary(&self.state.config.graph.language, &w),
8085                    detail,
8086                    gt::question_choices(&w, g.rewrite.as_ref()),
8087                );
8088                q.posting_comparison = Some(comparison);
8089                q.answer_timeout = timeout;
8090                // Recorded before it is written: a stop in between leaves a
8091                // record naming a question the resume then files again.
8092                g.question = Some(q.id.clone());
8093                g.asks += 1;
8094                self.state.github_text = Some(g.clone());
8095                self.state.event(
8096                    "github-text",
8097                    format!("asking the owner about the withheld text ({})", q.short()),
8098                );
8099                self.state.save()?;
8100                store
8101                    .put_executable(&mut q)
8102                    .context("file the posting-gate question")?;
8103                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8104                    tracing::warn!(
8105                        "could not notify about posting-gate question {}: {e:#}",
8106                        q.short()
8107                    );
8108                }
8109                self.park_on_github_text(&g)?;
8110                return Ok(None);
8111            }
8112        }
8113        let resolved = g.chosen_title.clone();
8114        let original = g.choice == Some(GithubTextChoice::Original);
8115        // Field names only: the event log never records the text.
8116        self.state.event(
8117            "github-text",
8118            if original {
8119                format!(
8120                    "posting the original text for: {}",
8121                    match (g.title, g.body) {
8122                        (true, true) => "title, description",
8123                        (true, false) => "title",
8124                        _ => "description",
8125                    }
8126                )
8127            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8128                "posting the saved rewrite title and description".to_owned()
8129            } else if resolved.is_some() {
8130                "posting the owner's replacement title".to_owned()
8131            } else {
8132                "posting the neutral text for the withheld fields".to_owned()
8133            },
8134        );
8135        self.state.github_text = Some(g.clone());
8136        self.state.save()?;
8137        if g.choice == Some(GithubTextChoice::Rewrite)
8138            && let Some(r) = g.rewrite.as_ref()
8139        {
8140            return Ok(Some(PrMessage {
8141                title: r.title.clone(),
8142                body: r.body.clone(),
8143            }));
8144        }
8145        let w = gt::Withheld {
8146            title: g.title,
8147            body: g.body,
8148            categories: g.categories.clone(),
8149        };
8150        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8151        if let Some(t) = resolved {
8152            title = t;
8153        }
8154        // Every withheld field is now neutral or vetted English.
8155        let english = gt::LanguageDecision {
8156            title_english: true,
8157            body_english: true,
8158            body_complete: true,
8159        };
8160        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8161        if original {
8162            // The owner's explicit choice overrides the gate for the fields it
8163            // was made for; the others keep what the gate produced.
8164            if g.title {
8165                title = pr.title.clone();
8166            }
8167            if g.body {
8168                body = pr.body.clone();
8169            }
8170        }
8171        Ok(Some(PrMessage { title, body }))
8172    }
8173
8174    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8175        self.state.github_text = Some(g.clone());
8176        self.state.end_pass_parked(ParkCause::Question);
8177        self.state.parked = true;
8178        self.state.event(
8179            "github-text",
8180            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8181        );
8182        self.state.save()
8183    }
8184
8185    /// Ask `[roles] language_judge` about this text and record which source
8186    /// decided; the text itself is never recorded.
8187    async fn judge_pr_language(
8188        &mut self,
8189        cwd: &Path,
8190        pr: &PrMessage,
8191    ) -> Option<crate::github_text::LanguageDecision> {
8192        self.state.config.roles.language_judge.as_ref()?;
8193        if !self.state.config.graph.language_gate() {
8194            return None;
8195        }
8196        let decision =
8197            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8198        self.state.event(
8199            "github-text",
8200            if decision.is_some() {
8201                "language decided by the language judge"
8202            } else {
8203                "language judge unavailable; using built-in heuristics"
8204            },
8205        );
8206        decision
8207    }
8208
8209    async fn merge(&mut self) -> Result<()> {
8210        // Same reasoning as `gate`: ask the review and gate records directly
8211        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8212        // skip can rewrite on reentry to something that no longer says
8213        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8214        // so a hand-off (open findings, green verification) reaches merge
8215        // exactly like a genuinely clean round does.
8216        //
8217        // A run resumed mid-`land` never reaches here at all: `execute`
8218        // recognises `RunStatus::Landing` before it even calls `prep`, and
8219        // routes straight to `run_land` instead. That has to happen a level
8220        // up from this function, not with a check in here, because
8221        // `review_loop`'s own status recomputation (see its doc) runs
8222        // *before* `merge` on every reentry and would otherwise overwrite
8223        // the `Landing` marker with `Gating` before this node ever saw it.
8224        if self
8225            .state
8226            .base_sync
8227            .as_ref()
8228            .is_some_and(|s| s.conflict.is_some())
8229            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8230                != Some(RunStatus::Gating)
8231            // `gate_ran == false` is not "passed" - `gate` leaves it false
8232            // both before it has ever run and when its last attempt was
8233            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8234            // permission to merge on nothing but the review record. Only a
8235            // gate that actually ran may proceed; an intentional local
8236            // review can also finish with verification explicitly not configured.
8237            // `RunState::gate_status` computes that distinction.
8238            || !self.state.gate_allows_progress()
8239        {
8240            return Ok(());
8241        }
8242        // This node's own record, not `status`: `status == Ready` is not
8243        // unique to the harmless `MergeMode::None` path this line was
8244        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8245        // run's PR was closed without merging — and on that run `mode` is
8246        // still `Pr`, so a reentry that fell through here would push and
8247        // open a second pull request. `self.state.merge` is set exactly once
8248        // this node (or `land`) has already produced a verdict, under every
8249        // mode, which is what "already done" actually means here.
8250        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8251            return Ok(());
8252        }
8253        let Some(winner) = self.state.winner().cloned() else {
8254            return Ok(());
8255        };
8256        let repo = self.state.repo.clone();
8257        let base = self.state.base_branch.clone();
8258        let mode = self.state.config.merge.mode;
8259        let style = self.state.config.merge.style;
8260        let facts = if is_review_run(&self.state) {
8261            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8262            let start = review_base(
8263                &repo,
8264                &self.state.config.merge.remote,
8265                &base,
8266                &self.state.base_commit,
8267                &winner.branch,
8268            )
8269            .await;
8270            branch_facts(&repo, &start, &winner.branch).await
8271        } else {
8272            None
8273        };
8274        // `None` when the base could not be freshly read: then an adopted
8275        // pull request's title is left alone.
8276        let leaked = if is_review_run(&self.state) {
8277            leaked_subjects(&self.state, &winner.branch).await
8278        } else {
8279            Some(Vec::new())
8280        };
8281        // `None`: parked on the owner's word about a withheld title or body.
8282        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8283        // the resume arrives here again.
8284        let Some(pr) = self
8285            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8286            .await?
8287        else {
8288            return Ok(());
8289        };
8290        let message = pr.commit_message();
8291
8292        let outcome = match mode {
8293            MergeMode::None => MergeOutcome {
8294                mode,
8295                ok: true,
8296                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8297                empty: false,
8298            },
8299            MergeMode::Pr | MergeMode::Local
8300                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8301            {
8302                MergeOutcome {
8303                    mode,
8304                    ok: false,
8305                    detail: empty_candidate_detail(&self.state, &base),
8306                    empty: true,
8307                }
8308            }
8309            MergeMode::Local => {
8310                let on = git::current_branch(&repo).await?;
8311                if on.as_deref() != Some(base.as_str()) {
8312                    MergeOutcome {
8313                        mode,
8314                        ok: false,
8315                        detail: format!(
8316                            "{} has {} checked out, not the base branch {base}",
8317                            repo.display(),
8318                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8319                        ),
8320                        empty: false,
8321                    }
8322                } else if !git::is_clean(&repo).await? {
8323                    MergeOutcome {
8324                        mode,
8325                        ok: false,
8326                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8327                        empty: false,
8328                    }
8329                } else {
8330                    let out = match style {
8331                        MergeStyle::Merge => {
8332                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8333                        }
8334                        MergeStyle::Squash => {
8335                            git::merge_squash(&repo, &winner.branch, &message).await?
8336                        }
8337                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8338                    };
8339                    MergeOutcome {
8340                        mode,
8341                        ok: out.ok(),
8342                        detail: if out.ok() { out.stdout } else { out.stderr },
8343                        empty: false,
8344                    }
8345                }
8346            }
8347            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8348                self.state.pr_publication_pending = true;
8349                self.state.save()?;
8350                let out =
8351                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8352                self.state.pr_publication_pending = out.is_err();
8353                MergeOutcome {
8354                    mode,
8355                    ok: out.is_ok(),
8356                    detail: match out {
8357                        Ok(url) => url,
8358                        Err(e) => format!("{e:#}"),
8359                    },
8360                    empty: false,
8361                }
8362            }
8363            MergeMode::Pr => {
8364                let remote = self.state.config.merge.remote.clone();
8365                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8366                if !pushed.ok() {
8367                    MergeOutcome {
8368                        mode,
8369                        ok: false,
8370                        detail: pushed.stderr,
8371                        empty: false,
8372                    }
8373                } else {
8374                    // A retry or resume of a run whose branch already has an
8375                    // open pull request adopts it rather than failing on a
8376                    // duplicate. Only this winner branch into this base:
8377                    // `branch_for` derives the name from the run id, so a
8378                    // different run's pull request never matches.
8379                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8380                    let out = match pr_merge_plan(found) {
8381                        PrPlan::Create => {
8382                            gh_pr_create(
8383                                &winner.worktree,
8384                                &base,
8385                                &winner.branch,
8386                                &pr.title,
8387                                &pr.body,
8388                            )
8389                            .await
8390                        }
8391                        PrPlan::Adopt { url, title } => {
8392                            self.state
8393                                .event("merge", format!("Pr: adopted open pull request {url}"));
8394                            if self.state.github_text.as_ref().is_some_and(|g| {
8395                                g.resolved
8396                                    && g.choice == Some(GithubTextChoice::Rewrite)
8397                                    && g.rewrite.is_some()
8398                            }) {
8399                                // The owner chose the entire pair, even if the
8400                                // existing title already matches or was edited.
8401                                land::set_pr_text_verbatim(
8402                                    &winner.worktree,
8403                                    &url,
8404                                    &pr.title,
8405                                    &pr.body,
8406                                )
8407                                .await
8408                                .map(|()| url)
8409                            } else {
8410                                if title != pr.title
8411                                    && (!is_review_run(&self.state)
8412                                        || leaked
8413                                            .as_deref()
8414                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8415                                    && let Err(e) =
8416                                        if self.state.github_text.as_ref().is_some_and(|g| {
8417                                            g.resolved
8418                                                && g.title
8419                                                && g.choice == Some(GithubTextChoice::Original)
8420                                        }) {
8421                                            // The owner chose to post this title unmodified.
8422                                            land::set_pr_title_verbatim(
8423                                                &winner.worktree,
8424                                                &url,
8425                                                &pr.title,
8426                                            )
8427                                            .await
8428                                        } else {
8429                                            land::set_pr_title(
8430                                                &mut self.state,
8431                                                &winner.worktree,
8432                                                &url,
8433                                                &pr.title,
8434                                            )
8435                                            .await
8436                                        }
8437                                {
8438                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8439                                    self.state
8440                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8441                                }
8442                                Ok(url)
8443                            }
8444                        }
8445                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8446                    };
8447                    match out {
8448                        Ok(url) => MergeOutcome {
8449                            mode,
8450                            ok: true,
8451                            detail: url,
8452                            empty: false,
8453                        },
8454                        Err(e) => MergeOutcome {
8455                            mode,
8456                            ok: false,
8457                            detail: e.to_string(),
8458                            empty: false,
8459                        },
8460                    }
8461                }
8462            }
8463        };
8464
8465        self.state.status = match (mode, outcome.ok) {
8466            (MergeMode::None, _) => RunStatus::Ready,
8467            (_, true) if outcome.empty => RunStatus::Ready,
8468            (MergeMode::Pr, true) => RunStatus::Ready,
8469            (_, true) => RunStatus::Merged,
8470            (_, false) => RunStatus::Blocked,
8471        };
8472        self.state.event(
8473            "merge",
8474            format!(
8475                "{:?}: {}",
8476                mode,
8477                outcome.detail.lines().next().unwrap_or("")
8478            ),
8479        );
8480        self.state.merge = Some(outcome);
8481        self.state.save()?;
8482
8483        // The PR is open and the run would historically stop here, leaving the
8484        // operator to watch checks, feed review comments back to a fixer, and
8485        // merge. That was done by hand six times in one session before this
8486        // existed. Opt-in, because merging is the one irreversible thing magi
8487        // can do to a repository.
8488        if self.state.config.graph.land
8489            && mode == MergeMode::Pr
8490            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8491        {
8492            self.run_land().await?;
8493        }
8494        // `run_land` may have left `status` at `Landing` - still waiting on
8495        // CI or the owner's approval, not actually settled - so this has to
8496        // read whatever `status` ended up as here, not the `Merged` this
8497        // function set a few lines up.
8498        self.settle_questions();
8499        Ok(())
8500    }
8501
8502    /// Enter `land`.
8503    ///
8504    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8505    /// a resumed run's re-entry. `land::land` itself is what serialises the
8506    /// two git-mutating moments inside the loop — the rebase push and
8507    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8508    /// to hold a lock across the whole call, and doing so would serialise
8509    /// this run's CI wait against a *different* run's land-approval resume
8510    /// in the same repository, which is exactly the "must not wait on
8511    /// another task" property the daemon's slot-freeing exists to give.
8512    async fn run_land(&mut self) -> Result<()> {
8513        crate::pr_head::ensure_owner(&self.state)?;
8514        let url = self
8515            .state
8516            .pr
8517            .as_ref()
8518            .map(|pr| pr.url.clone())
8519            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8520            .unwrap_or_default();
8521        let url = url.lines().next().unwrap_or("").trim().to_owned();
8522        if !url.starts_with("http") {
8523            return Ok(());
8524        }
8525        // A land failure is not a lost run: the work is on a branch and the
8526        // pull request is open, which is exactly where a human takes over.
8527        loop {
8528            match land::land(&mut self.state, &url).await {
8529                Ok(_)
8530                    if self
8531                        .state
8532                        .merge_repairs
8533                        .last()
8534                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8535                {
8536                    if self.repair_merge_review().await? {
8537                        continue;
8538                    }
8539                }
8540                Ok(pr) if self.state.parked => {
8541                    // `land` already saved the parked marker; nothing here
8542                    // overrides `status` back to a terminal value while an
8543                    // approval is still outstanding.
8544                    let _ = pr;
8545                }
8546                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8547                    self.state.status = RunStatus::Superseded;
8548                    self.state.save()?;
8549                }
8550                Ok(pr) => {
8551                    self.state.status = match pr.state {
8552                        land::PrLifecycle::Merged => RunStatus::Merged,
8553                        _ => RunStatus::Blocked,
8554                    };
8555                    // Downstream of a confirmed merge only - see
8556                    // `bump::should_release_bump`'s own doc for why this one
8557                    // check covers all three of `land`'s success paths.
8558                    // Best-effort: the run already landed, so a failure here
8559                    // (the decision call, `gh`, `cargo`) is recorded and never
8560                    // turns a landed run into a failed one.
8561                    if bump::should_release_bump(self.state.status)
8562                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8563                    {
8564                        // The event is the run's own record. Not-eligible cases
8565                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8566                        // `Err` is a bump that was tried and failed:
8567                        // `after_merge` itself raises the operator notice for
8568                        // that, whether or not a release PR exists yet.
8569                        self.state
8570                            .event("bump", format!("release bump skipped: {e:#}"));
8571                    }
8572                    // Independent of the bump, and best-effort in the same way:
8573                    // findings the merge left open become follow-up tasks.
8574                    if self.state.status == RunStatus::Merged {
8575                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8576                    }
8577                    self.state.save()?;
8578                }
8579                Err(e) => {
8580                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8581                        return Err(e);
8582                    }
8583                    self.state.status = RunStatus::Blocked;
8584                    self.state.event("land", format!("gave up: {e}"));
8585                    self.state.save()?;
8586                }
8587            }
8588            break;
8589        }
8590        Ok(())
8591    }
8592
8593    /// One repair cycle stays under the existing driver/task claim. An
8594    /// interrupted fixer is rolled back and held rather than run twice.
8595    async fn repair_merge_review(&mut self) -> Result<bool> {
8596        use crate::run::MergeRepairPhase as Phase;
8597        crate::pr_head::ensure_owner(&self.state)?;
8598        let i = self.state.merge_repairs.len() - 1;
8599        let request = self.state.merge_repairs[i].clone();
8600        if request.phase == Phase::Fixing {
8601            let restored = match self.state.winner().cloned() {
8602                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8603                None => Err(anyhow::anyhow!("repair has no winning branch")),
8604            };
8605            let why = match restored {
8606                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8607                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8608                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8609            };
8610            self.hold_merge_repair(i, why)?;
8611            return Ok(false);
8612        }
8613        let failure = if i >= self.state.config.graph.land_rounds
8614            || self.state.config.graph.review_rounds == 0
8615        {
8616            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8617        } else {
8618            None
8619        };
8620        if let Some(why) = failure {
8621            self.hold_merge_repair(i, why)?;
8622            return Ok(false);
8623        }
8624        if request.phase == Phase::Requested {
8625            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8626                Ok(pr) => pr,
8627                Err(e) => {
8628                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8629                    return Ok(false);
8630                }
8631            };
8632            self.state.merge_repairs[i].phase = Phase::Fixing;
8633            self.state.land_approval = None;
8634            self.state.land_armed_head = None;
8635            self.state.save()?;
8636            // Serialize the exact records, including finding ids/details,
8637            // reconsideration reasons, final votes and verification output.
8638            let evidence = serde_json::to_string_pretty(&request)?;
8639            let reason = format!(
8640                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8641            );
8642            let budget = self.state.config.graph.land_rounds;
8643            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8644            match result {
8645                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8646                Ok(land::Fixed::Failed(why)) => {
8647                    self.hold_merge_repair(
8648                        i,
8649                        format!(
8650                            "repair failed: {why}; inspect the branch and run an explicit review"
8651                        ),
8652                    )?;
8653                    return Ok(false);
8654                }
8655                Err(e) => {
8656                    self.hold_merge_repair(
8657                        i,
8658                        format!(
8659                            "repair failed: {e:#}; inspect the branch and run an explicit review"
8660                        ),
8661                    )?;
8662                    return Ok(false);
8663                }
8664            }
8665            // Even no change gets a real new reviewer wave. The previous
8666            // evidence remains in the request; a former clean/concluded round
8667            // cannot short-circuit the new review or gate.
8668            self.begin_merge_repair_review(i)?;
8669        }
8670        self.review_loop().await?;
8671        self.gate().await?;
8672        if self.resource_halt()? || self.state.parked {
8673            return Ok(false);
8674        }
8675        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8676            != Some(RunStatus::Gating)
8677            || !self.state.gate_allows_progress()
8678        {
8679            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8680            return Ok(false);
8681        }
8682        // Review and gate may have added commits after the initial repair.
8683        // Publish exactly their final head to the existing PR, with a lease.
8684        let winner = self
8685            .state
8686            .winner()
8687            .cloned()
8688            .context("repair has no winning branch")?;
8689        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8690        if self.state.gate_head.as_deref() != Some(after.as_str())
8691            || !git::is_clean(&winner.worktree).await?
8692        {
8693            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8694            return Ok(false);
8695        }
8696        let expected = self
8697            .state
8698            .pr_pushed_head
8699            .as_deref()
8700            .unwrap_or(&request.head)
8701            .to_owned();
8702        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8703        // Push the checked commit itself: a local ref moving after the check
8704        // must not change which commit the forge receives.
8705        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8706        let refspec = format!("{after}:refs/heads/{public}");
8707        let push = git::git_raw(
8708            &winner.worktree,
8709            &["push", &lease, &self.state.config.merge.remote, &refspec],
8710        )
8711        .await;
8712        match push {
8713            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8714            Ok(out) => {
8715                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8716                return Ok(false);
8717            }
8718            Err(e) => {
8719                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8720                return Ok(false);
8721            }
8722        }
8723        self.state.merge_repairs[i].phase = Phase::Complete;
8724        self.state.status = RunStatus::Landing;
8725        self.state.event(
8726            "repair",
8727            "new review and gate completed; fresh owner merge decision required",
8728        );
8729        self.state.save()?;
8730        Ok(true)
8731    }
8732
8733    fn merge_repair_artifacts(&self) -> PathBuf {
8734        let artifacts = agent::artifacts_dir(&self.state.dir());
8735        match self
8736            .state
8737            .merge_repairs
8738            .last()
8739            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8740        {
8741            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8742            None => artifacts,
8743        }
8744    }
8745
8746    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8747        self.state.reviews.clear();
8748        self.state.gate.clear();
8749        self.state.gate_ran = false;
8750        self.state.gate_head = None;
8751        self.state.gate_fixes.clear();
8752        self.state.gate_repair_e2e = None;
8753        self.state.pre_gate.clear();
8754        self.state.pre_gate_commit = None;
8755        self.state.contested_handoff = None;
8756        self.state.land_reruns = None;
8757        self.state.land_approval = None;
8758        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8759        self.state.status = RunStatus::Reviewing;
8760        self.state.save()
8761    }
8762
8763    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8764        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8765        self.state.merge_repairs[index].failure = Some(why.clone());
8766        self.state.status = RunStatus::Blocked;
8767        self.state.event("repair", why);
8768        self.state.save()
8769    }
8770
8771    // -------------------------------------------------------------- helpers
8772
8773    /// Fetch or create a seat, keeping its conversation across nodes.
8774    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8775        if let Some(existing) = self.state.seats.get(key)
8776            && existing.agent == agent
8777        {
8778            return existing.clone();
8779        }
8780        // A seat that changes agent mints its session id from the agent too,
8781        // like a handover: the old agent's uuid is already taken by the CLI.
8782        let fresh = if self.state.seats.contains_key(key) {
8783            handover_seat(key, agent, self.state.next_seat_seed())
8784        } else {
8785            SeatState::new(key, agent, self.state.seed)
8786        };
8787        self.state.seats.insert(key.to_owned(), fresh.clone());
8788        fresh
8789    }
8790
8791    /// The agent now holding seat `key`: `spec`, unless a handover moved the
8792    /// seat to another roster agent, in which case that agent. Nodes that
8793    /// continue a seat's conversation (deliberation, the votes, a reviewer's
8794    /// reconsideration) must keep talking to whoever answered it, not slip
8795    /// back to the agent that failed it.
8796    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8797        match self.state.seats.get(key) {
8798            Some(s) if s.agent != spec.id => {
8799                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8800            }
8801            _ => spec,
8802        }
8803    }
8804
8805    /// A candidate rendered for judging, with the leak policy applied.
8806    fn view(&self, c: &Candidate) -> CandidateView {
8807        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8808            .unwrap_or_default();
8809        let (patch, _) = blind::sanitize_patch(
8810            &format!("candidate {} patch", c.label),
8811            &raw,
8812            &self.state.config.blind,
8813        );
8814        CandidateView {
8815            label: c.label,
8816            branch: c.branch.clone(),
8817            summary: c.summary.clone(),
8818            stat: c.stat.clone(),
8819            patch,
8820        }
8821    }
8822
8823    /// The full candidate set as prompt text, for seats with no live session.
8824    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8825        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8826        prompt::judge(
8827            "(see above)",
8828            &views,
8829            self.roles.judges.len(),
8830            base_short,
8831            "en",
8832        )
8833    }
8834
8835    /// The final-vote prompt with everything a seat that has no session of its
8836    /// own needs: the candidates, the seat's own ranking and reasons, and the
8837    /// anonymised deliberation it took part in (only when there was one, so a
8838    /// handed-over seat never sees more than the seat it replaces did). The
8839    /// `Final vote` heading stays first.
8840    fn vote_prompt_full(
8841        &self,
8842        j: usize,
8843        labels: &[char],
8844        language: &str,
8845        candidates: &[Candidate],
8846        base_short: &str,
8847    ) -> String {
8848        let mut text = format!(
8849            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8850            prompt::final_vote(labels, language),
8851            self.state.instruction,
8852            self.candidate_block(candidates, base_short)
8853        );
8854        if let Some(own) = self
8855            .state
8856            .judgements
8857            .get(j)
8858            .filter(|r| !r.ranking.is_empty())
8859        {
8860            let reasons = own
8861                .reasons
8862                .iter()
8863                .map(|(k, v)| format!("- {k}: {v}"))
8864                .collect::<Vec<_>>()
8865                .join("\n");
8866            text.push_str(&format!(
8867                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8868                own.ranking.iter().collect::<String>(),
8869                if reasons.is_empty() {
8870                    ""
8871                } else {
8872                    ", because:\n"
8873                }
8874            ));
8875        }
8876        if !self.state.deliberation.is_empty() {
8877            text.push_str("\n# What was argued before this vote\n");
8878            for t in self.transcript(&[], j) {
8879                text.push_str(&format!(
8880                    "\n## {}{}\n\n{}\n",
8881                    t.who,
8882                    if t.is_self { " (you)" } else { "" },
8883                    t.body.trim()
8884                ));
8885            }
8886        }
8887        text
8888    }
8889
8890    /// Anonymised transcript for judge `self_idx`.
8891    ///
8892    /// The initial rankings are always the opening statements. Seeding them
8893    /// only when no turn had been taken yet meant every judge after the first
8894    /// argued against a single voice instead of against the actual split — the
8895    /// disagreement is the information, so it is always on the table.
8896    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8897        let mut turns = Vec::new();
8898        for j in &self.state.judgements {
8899            if j.ranking.is_empty() {
8900                continue;
8901            }
8902            let reasons = j
8903                .reasons
8904                .iter()
8905                .map(|(k, v)| format!("- {k}: {v}"))
8906                .collect::<Vec<_>>()
8907                .join("\n");
8908            turns.push(Turn {
8909                who: format!("Judge {} (opening ranking)", j.judge),
8910                is_self: j.judge == self_idx + 1,
8911                body: format!(
8912                    "Ranked {}{}{reasons}",
8913                    j.ranking.iter().collect::<String>(),
8914                    if reasons.is_empty() {
8915                        ""
8916                    } else {
8917                        ", because:\n"
8918                    }
8919                ),
8920            });
8921        }
8922        for t in self
8923            .state
8924            .deliberation
8925            .iter()
8926            .flat_map(|r| r.turns.iter())
8927            .chain(current)
8928        {
8929            turns.push(Turn {
8930                who: format!("Judge {}", t.judge),
8931                is_self: t.judge == self_idx + 1,
8932                body: t.body.clone(),
8933            });
8934        }
8935        turns
8936    }
8937}
8938
8939/// Does this seat still hold the context a follow-up prompt would rely on?
8940fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8941    agent::has_session(spec.kind, seat, sessions)
8942}
8943
8944/// The next entry in `roster` after `start`, never wrapping back to the
8945/// front, whose id is not in `tried` yet.
8946///
8947/// Starts one past `start` rather than at the front of `roster`: `start` is
8948/// the seat's own original position, and a seat whose candidate slot already
8949/// sits on the roster's second entry must fall through to the third next, not
8950/// restart at the first — which is very likely a different candidate's own
8951/// agent already. Never wraps back past `start`, for the same reason: an
8952/// entry earlier in the roster than the seat's own position is almost
8953/// certainly some *other* candidate slot's own agent, and once the tail of
8954/// the roster is exhausted there are no more untried agents for *this* seat
8955/// to fall through to — the caller's fallback chain ends there, exactly as
8956/// "no further untried agents remain in the list for that seat" asks for.
8957///
8958/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
8959/// the same id twice (an operator's `roles.implementers` typo, or a
8960/// `[[agents]]` list reused across roles) must not let
8961/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
8962/// over `roster` either finds an untried id or runs out, so this always
8963/// terminates regardless of duplicates.
8964fn next_untried_in_roster<'a>(
8965    roster: &'a [AgentSpec],
8966    start: usize,
8967    tried: &BTreeSet<String>,
8968) -> Option<&'a AgentSpec> {
8969    roster
8970        .get(start + 1..)?
8971        .iter()
8972        .find(|s| !tried.contains(&s.id))
8973}
8974
8975/// The successor for a seat, shared by all four seat kinds (implement, judge,
8976/// review, advise). `others` holds the ids that *currently* occupy the other
8977/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
8978/// sees them), so roster entries beyond the seat count act as spares: a failed
8979/// seat goes to an agent no other seat holds whenever the roster permits.
8980///
8981/// `carried` is `Some` only for the review loop's carried failure history.
8982/// Order: (1) forward from `start`, never wrapping, untried, not a carried
8983/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
8984/// whole roster: untried and not another seat's occupant; (3) the plain walk
8985/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
8986/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
8987/// a better outcome than an empty seat, and it keeps the answer to "is there
8988/// a successor at all" exactly what it was before occupants were considered,
8989/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
8990/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
8991/// carried failure is a cheaper bet than doubling an agent on the panel, and
8992/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
8993/// the same round are handled one at a time, so a seat later in the batch
8994/// sees an earlier one's new occupant but not yet its own freed agent.
8995fn pick_successor<'a>(
8996    roster: &'a [AgentSpec],
8997    start: usize,
8998    tried: &BTreeSet<String>,
8999    carried: Option<&BTreeSet<String>>,
9000    others: &BTreeSet<String>,
9001) -> Option<&'a AgentSpec> {
9002    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9003    roster
9004        .get(start + 1..)
9005        .and_then(|tail| {
9006            tail.iter()
9007                .filter(free)
9008                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9009        })
9010        .or_else(|| {
9011            carried
9012                .is_some()
9013                .then(|| roster.iter().find(free))
9014                .flatten()
9015        })
9016        .or_else(|| match carried {
9017            Some(c) => next_for_seat(roster, start, tried, c),
9018            None => next_untried_in_roster(roster, start, tried),
9019        })
9020}
9021
9022/// The next agent for a seat that carries its failure history across rounds
9023/// (the review loop). `round_tried` is this round's own bound and starts
9024/// empty every round; `carried_failed` only decides priority.
9025///
9026/// First: an id walking forward from `start`, never wrapping, that is neither
9027/// tried this round nor failed in an earlier one. Only when that is exhausted
9028/// does it rescue: the first roster id (in roster order, so this one *does*
9029/// look before `start`) not yet tried this round, which is by then a carried
9030/// failure. Each id is rescued at most once per round, so it cannot loop.
9031fn next_for_seat<'a>(
9032    roster: &'a [AgentSpec],
9033    start: usize,
9034    round_tried: &BTreeSet<String>,
9035    carried_failed: &BTreeSet<String>,
9036) -> Option<&'a AgentSpec> {
9037    roster
9038        .get(start + 1..)?
9039        .iter()
9040        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9041        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9042}
9043
9044/// Where a reviewer seat starts a round: the agent that last answered it when
9045/// it is still on the roster and not marked failed, else the spec's own agent
9046/// unless it failed, else the next roster agent that has not failed, else the
9047/// spec's own agent again (the whole roster failed). Ids no longer on the
9048/// roster are ignored. An empty roster has no handover, so the spec stands.
9049fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9050    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9051        return spec;
9052    };
9053    let ok = |id: &str| !h.failed.contains(id);
9054    if let Some(last) = h.last_ok.as_deref()
9055        && ok(last)
9056        && let Some(s) = roster.iter().find(|s| s.id == last)
9057    {
9058        return s.clone();
9059    }
9060    if ok(&spec.id) {
9061        return spec;
9062    }
9063    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9064    roster
9065        .get(from + 1..)
9066        .into_iter()
9067        .flatten()
9068        .chain(roster.iter())
9069        .find(|s| ok(&s.id))
9070        .cloned()
9071        .unwrap_or(spec)
9072}
9073
9074/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
9075/// seed so a CLI that mints its session id up front (`--session-id`) never
9076/// reuses the uuid the previous agent already opened under the same seat key.
9077pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9078    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9079}
9080
9081/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
9082/// so the classifier below and the code that builds the message cannot drift.
9083const TIMED_OUT: &str = "timed out";
9084
9085impl FailClass {
9086    /// `None` for an answer; otherwise how the turn failed.
9087    fn of(out: &AgentOutcome) -> Option<Self> {
9088        match out {
9089            AgentOutcome::Ok(_) => None,
9090            AgentOutcome::Quota(_) => Some(Self::Quota),
9091            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9092            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9093            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9094            // Not a failure of the agent. Callers check for it first; `None`
9095            // here means "no class to hand over on".
9096            AgentOutcome::ResourceBlocked(_) => None,
9097        }
9098    }
9099
9100    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
9101    fn stem_word(&self) -> &'static str {
9102        match self {
9103            Self::Quota => "quota",
9104            _ => "handover",
9105        }
9106    }
9107}
9108
9109/// The message's first line with its variable parts removed — digit runs and
9110/// path-like tokens — so "exit code 2" and "exit code 7" read
9111/// as one kind of failure.
9112fn failure_signature(msg: &str) -> String {
9113    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9114    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
9115    // the historical machine key while the process diagnosis uses readable prose.
9116    if let Some(code) = line
9117        .strip_prefix("process exited with no usable output (exit code ")
9118        .and_then(|s| s.strip_suffix(')'))
9119        .and_then(|s| s.parse::<i32>().ok())
9120    {
9121        line = format!("exited with some({code}) and no usable output");
9122    } else if matches!(
9123        line.as_str(),
9124        "process exited with no usable output (no exit code reported)"
9125            | "process exited with no usable output (exit code unavailable)"
9126    ) {
9127        line = "exited with none and no usable output".to_owned();
9128    }
9129    let mut out = Vec::new();
9130    for word in line.split_whitespace() {
9131        if word.contains('/') || word.contains('\\') {
9132            out.push("<path>".to_owned());
9133            continue;
9134        }
9135        let mut w = String::new();
9136        let mut in_digits = false;
9137        for c in word.chars() {
9138            if c.is_ascii_digit() {
9139                if !in_digits {
9140                    w.push('#');
9141                }
9142                in_digits = true;
9143            } else {
9144                in_digits = false;
9145                w.push(c);
9146            }
9147        }
9148        out.push(w);
9149    }
9150    out.join(" ").chars().take(120).collect()
9151}
9152
9153/// Whether a seat that just failed with `cur` may go to the next roster agent.
9154/// A quota or a timeout always may. Any other failure may not when the agent
9155/// before it failed the same way: an error the prompt causes would otherwise
9156/// walk the whole roster. `prev` is the class of the immediately preceding
9157/// agent's failure, so a quota or timeout in between breaks the run of
9158/// identical failures by itself.
9159fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9160    match cur {
9161        FailClass::Quota | FailClass::Timeout => true,
9162        FailClass::Other(_) => prev != Some(cur),
9163    }
9164}
9165
9166/// A short human reason for a failed outcome, for the handover record.
9167fn fail_reason(out: &AgentOutcome) -> String {
9168    match out {
9169        AgentOutcome::Ok(_) => String::new(),
9170        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9171        AgentOutcome::Dropped(o) => format!(
9172            "the CLI dropped the stream ({})",
9173            o.dropped
9174                .as_ref()
9175                .map(|d| d.why.as_str())
9176                .unwrap_or("it ended without delivering its answer")
9177        ),
9178        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9179            e.lines().next().unwrap_or("").chars().take(160).collect()
9180        }
9181    }
9182}
9183
9184/// Note one handover in the run: the structured record and, in the timeline,
9185/// the sentence a person reads. A quota keeps the wording it always had.
9186pub(crate) fn record_handover(
9187    state: &mut RunState,
9188    node: &str,
9189    seat: &str,
9190    from: &str,
9191    to: &str,
9192    class: &FailClass,
9193    reason: &str,
9194) {
9195    let message = if *class == FailClass::Quota {
9196        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9197    } else {
9198        format!("{seat}: handed over {from} -> {to} ({reason})")
9199    };
9200    state.event(node, message);
9201    state.handovers.push(Handover {
9202        at: Timestamp::now(),
9203        node: node.to_owned(),
9204        seat: seat.to_owned(),
9205        from: from.to_owned(),
9206        to: to.to_owned(),
9207        reason: reason.to_owned(),
9208    });
9209}
9210
9211/// Did this reply report running a command whose own CLI never confirmed an
9212/// exit status?
9213///
9214/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9215/// command *finished* (see that type's own doc), so this can only be `true`
9216/// for a command whose completion event carried no readable exit code — not
9217/// for one that simply is not mentioned at all. That is the one signal this
9218/// crate can read, from the same record `state.jobs` renders, about a reply
9219/// standing next to work its own CLI cannot vouch for finishing; it is
9220/// deliberately not a check on the exit code's *value* (a fixer legitimately
9221/// runs a command that fails mid-iteration before it succeeds) and not a
9222/// guess at a command still running in the background (which emits no event
9223/// at all, and so leaves no evidence here to find).
9224fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9225    commands.iter().any(|c| c.exit_code.is_none())
9226}
9227
9228/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9229/// trusted as a verified no-op — the adoption guard's own text-level half.
9230///
9231/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9232/// timed out): a marker only earns the benefit of the doubt from a turn the
9233/// CLI itself vouches for finishing properly, the same house style
9234/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9235/// *fix* report to for `commands`. A candidate that timed out, exited
9236/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9237/// is, whatever prose it wrote — this returns `None` before it ever looks at
9238/// `text`. The remaining guards (the tree really is empty, the evidence is
9239/// non-empty) are the caller's: this only reads what the reply *claimed*.
9240fn verified_noop_claim(
9241    usable: bool,
9242    commands: &[agent::CommandEvidence],
9243    text: &str,
9244) -> Option<String> {
9245    (usable && !has_unconfirmed_command(commands))
9246        .then(|| verdict::verified_noop(text))
9247        .flatten()
9248}
9249
9250fn short(commit: &str) -> String {
9251    commit.chars().take(7).collect()
9252}
9253
9254fn make_executable(path: &Path) -> Result<()> {
9255    #[cfg(unix)]
9256    {
9257        use std::os::unix::fs::PermissionsExt as _;
9258        let mut perms = std::fs::metadata(path)?.permissions();
9259        perms.set_mode(0o755);
9260        std::fs::set_permissions(path, perms)?;
9261    }
9262    #[cfg(not(unix))]
9263    {
9264        let _ = path;
9265    }
9266    Ok(())
9267}
9268
9269/// What every seat in one batch shares: where the answers are attributed, the
9270/// prompt overlay they inherit, and the build cache they are told to use.
9271///
9272/// A struct rather than four more parameters: `wave` also needs the run's
9273/// state (to record who is answering right now) and the attempt number, and
9274/// eight positional arguments is both unreadable and a clippy error.
9275struct WaveCtx<'a> {
9276    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9277    /// paid for it.
9278    run: &'a str,
9279    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9280    node: &'a str,
9281    prompts: &'a Prompts,
9282    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9283    cache: Option<&'a Path>,
9284    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9285    /// `JobRecord::round`. `None` for every other node.
9286    round: Option<usize>,
9287    /// Carry each seat's failed-agent history across waves (the review loop
9288    /// only): start-of-round priority and handover choice read
9289    /// [`RunState::seat_history`], and every answer or failure writes it.
9290    carry_seats: bool,
9291    /// The run's park / interrupt requests, so a wait for a shared resource
9292    /// can be abandoned the moment the operator asks to park.
9293    pauses: Option<(&'a Pause, &'a Pause)>,
9294}
9295
9296/// Shared scheduling policy for operations and spawned seat invocations.
9297fn repository_wait(
9298    state: &RunState,
9299    pauses: Option<(&Pause, &Pause)>,
9300) -> crate::repo_lock::WaitPolicy {
9301    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9302    let inherited = crate::repo_lock::WaitPolicy::current();
9303    crate::repo_lock::WaitPolicy::new(
9304        Duration::from_secs(state.config.graph.timeout_resource_wait),
9305        move || {
9306            inherited.cancelled()
9307                || pauses
9308                    .as_ref()
9309                    .is_some_and(|(a, b)| a.parked() || b.parked())
9310        },
9311    )
9312}
9313
9314fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9315    if error.is::<crate::repo_lock::ResourceBlocked>() {
9316        state.resource_block = Some(crate::run::ResourceBlock {
9317            node: "repository".into(),
9318            kind: if error.to_string().contains("disk pressure") {
9319                "disk pressure"
9320            } else {
9321                "repository coordination"
9322            }
9323            .into(),
9324            detail: error.to_string(),
9325            at: Timestamp::now(),
9326        });
9327        state.event("resource", error.to_string());
9328    }
9329}
9330
9331async fn coordinate_operation(
9332    state: &mut RunState,
9333    cwd: &Path,
9334    wait: &crate::repo_lock::WaitPolicy,
9335) -> Result<Option<crate::repo_lock::SeatLease>> {
9336    let result =
9337        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9338            .await;
9339    if let Err(e) = &result {
9340        record_repository_block(state, e);
9341    }
9342    result
9343}
9344
9345/// Run one job, honouring the parallelism budget.
9346async fn run_one(
9347    job: SeatJob,
9348    sem: Arc<Semaphore>,
9349    ctx: &WaveCtx<'_>,
9350    state: &mut RunState,
9351    attempt: usize,
9352) -> (SeatState, AgentOutcome) {
9353    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9354        .await
9355        .pop()
9356        .expect("one job in, one result out");
9357    (seat, out)
9358}
9359
9360/// Run every job concurrently, capped by the semaphore, preserving order.
9361///
9362/// Every seat in the batch is recorded into [`RunState::active`] before the
9363/// wave starts and cleared as each answer lands, so the run's own record says
9364/// who is still being waited on rather than only who finished.
9365async fn wave(
9366    jobs: Vec<SeatJob>,
9367    sem: Arc<Semaphore>,
9368    ctx: &WaveCtx<'_>,
9369    state: &mut RunState,
9370    attempt: usize,
9371) -> Vec<(usize, SeatState, AgentOutcome)> {
9372    let WaveCtx {
9373        run,
9374        node,
9375        prompts,
9376        cache,
9377        round,
9378        carry_seats: _,
9379        pauses,
9380    } = *ctx;
9381    for job in &jobs {
9382        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9383    }
9384    if let Err(e) = state.save() {
9385        // A failed persist of "who is answering right now" must not abort the
9386        // wave: the seats are already being asked, and the alternative is
9387        // losing the answers to save a status line nobody may even be
9388        // watching.
9389        tracing::warn!("could not persist in-progress seats: {e:#}");
9390    }
9391    // Hold the shared build cache's lease for the whole batch, not per job:
9392    // several candidates (an implement wave) or a fixer legitimately share
9393    // one cache concurrently within this run, and that stays untouched — a
9394    // single lease taken once for the whole wave and released once it is
9395    // done is what stops a *different* borrower (another run's own wave, its
9396    // e2e/gate, a human's `magi review`) from interleaving a build into the
9397    // same directory while this one is in flight. Best-effort, not
9398    // all-or-nothing: a wave that cannot get the lease within its own
9399    // longest job's budget still runs — an hour of paid implementer calls is
9400    // not thrown away over cache contention — but every write-allowed seat
9401    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9402    // below), the same fallback a read-only seat always gets, rather than
9403    // building into a directory this run was never granted. The identity
9404    // record is still invalidated below either way, so the next tracked
9405    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9406    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9407    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9408    // never borrowed from a job's agent budget (and never charged to it either,
9409    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9410    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9411    let cache_guard = if let Some(cache_dir) = cache {
9412        if jobs_had_a_writer {
9413            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9414            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9415            match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9416            {
9417                Ok(Some(g)) => Some(g),
9418                // Parking: leave without spawning anything. The run parks at
9419                // the next boundary and the seats are asked afresh on resume.
9420                Ok(None) => {
9421                    state.resource_block = Some(crate::run::ResourceBlock {
9422                        node: node.to_owned(),
9423                        kind: "shared build cache".to_owned(),
9424                        detail: "the run is parking".to_owned(),
9425                        at: Timestamp::now(),
9426                    });
9427                    return block_wave(
9428                        jobs,
9429                        state,
9430                        node,
9431                        attempt,
9432                        "parked while waiting for the shared build cache",
9433                    );
9434                }
9435                // An unreadable lease is not contention: keep the old
9436                // fallback (no shared cache for this wave's seats).
9437                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9438                Err(e) => {
9439                    // Writing seats may not build into a cache this run does
9440                    // not hold, and an agent that cannot be given one is not
9441                    // asked at all: the run stops resumably instead.
9442                    let why = format!("{e:#}");
9443                    state.resource_block = Some(crate::run::ResourceBlock {
9444                        node: node.to_owned(),
9445                        kind: "shared build cache".to_owned(),
9446                        detail: why.clone(),
9447                        at: Timestamp::now(),
9448                    });
9449                    state.event(
9450                        "resource",
9451                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9452                    );
9453                    return block_wave(jobs, state, node, attempt, &why);
9454                }
9455            }
9456        } else {
9457            None
9458        }
9459    } else {
9460        None
9461    };
9462    // Every job keeps its whole budget: the wait above was not the agent's.
9463    // Restart the seats' clocks so "started" means "being spawned now".
9464    state.seats_spawning(node, attempt);
9465    if let Err(e) = state.save() {
9466        tracing::warn!("could not persist the start of a wave: {e:#}");
9467    }
9468    // Queueing behind this wave's own seats for a parallelism slot is normal
9469    // (each of them is bounded by its own agent timeout); only a wait beyond
9470    // all of them plus the resource limit means the slots never came back.
9471    let slot_limit = resource_limit
9472        + jobs
9473            .iter()
9474            .map(|j| j.timeout)
9475            .max()
9476            .unwrap_or_default()
9477            .saturating_mul(jobs.len() as u32);
9478    let mut set = tokio::task::JoinSet::new();
9479    // Seats announce themselves here the moment they hold a parallelism
9480    // permit, i.e. just before their CLI is spawned.
9481    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9482    let overlay = prompts.overlay(node);
9483    let repository_wait = repository_wait(state, pauses);
9484    for (i, mut job) in jobs.into_iter().enumerate() {
9485        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9486        if cache.is_some() {
9487            job.prompt.push('\n');
9488            job.prompt
9489                .push_str(&prompt::build_cache_note(node, job.allow_write));
9490        }
9491        let sem = Arc::clone(&sem);
9492        let run = run.to_owned();
9493        let node = node.to_owned();
9494        // Only implementers were told about the task's attachments, so only
9495        // their seats get the directory widened for reading.
9496        let attachments = if node == "implement" {
9497            state.attachments.clone()
9498        } else {
9499            Vec::new()
9500        };
9501        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9502        // `prompt::build_cache_note`'s doc for why setting it anyway is
9503        // exactly how a sandboxed reviewer's write refusal got reported as a
9504        // defect in the patch, not a property of its own seat. And a
9505        // write-allowed one is handed it only when the lease above was
9506        // actually acquired: a wave that could not get it (`cache_guard` is
9507        // `None`, see its own comment) must not send seats to build into a
9508        // directory this run does not hold - that is the exact concurrent,
9509        // unmanaged-write race this module exists to prevent, not something
9510        // "proceeding anyway" is allowed to reintroduce.
9511        let cache = cache
9512            .filter(|_| job.allow_write && cache_guard.is_some())
9513            .map(Path::to_path_buf);
9514        let spawned_tx = spawned_tx.clone();
9515        let repository_wait = repository_wait.clone();
9516        let coordination = state.config.coordination.clone();
9517        let fetched_base = state.fetched_base.clone();
9518        set.spawn(async move {
9519            // A wait for a parallelism slot is scheduling, not the agent: it
9520            // is bounded by its own limit and ends as a resource block, never
9521            // as a timeout of the agent that was not started.
9522            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9523                Ok(p) => p,
9524                Err(_) => {
9525                    return (
9526                        i,
9527                        job.seat,
9528                        AgentOutcome::ResourceBlocked(format!(
9529                            "no parallel slot became free within {}s",
9530                            slot_limit.as_secs()
9531                        )),
9532                        false,
9533                    );
9534                }
9535            };
9536            spawned_tx.send(job.seat.key.clone()).ok();
9537            let mut seat = job.seat;
9538            let out = repository_wait
9539                .scope(agent::invoke(
9540                    &job.spec,
9541                    &mut seat,
9542                    &Invocation {
9543                        scope: crate::agent::InvocationScope::Repository,
9544                        coordination: coordination.as_ref(),
9545                        fetched_base: fetched_base.as_ref(),
9546                        cwd: &job.cwd,
9547                        prompt: &job.prompt,
9548                        timeout: job.timeout,
9549                        allow_write: job.allow_write,
9550                        unsandboxed: false,
9551                        sessions: job.sessions,
9552                        artifacts: &job.artifacts,
9553                        stem: &job.stem,
9554                        run: &run,
9555                        node: &node,
9556                        cache_dir: cache.as_deref(),
9557                        attachments: &attachments,
9558                        writable: &[],
9559                    },
9560                ))
9561                .await;
9562            let interrupted = out
9563                .as_ref()
9564                .err()
9565                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9566            let out = match out {
9567                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9568                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9569                // Billed work the CLI failed to hand over is not an ordinary
9570                // failure, but its text is the CLI's raw error JSON, not an
9571                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9572                // read it as one by forgetting to check. `usable()` is always
9573                // false here (dropped implies an empty response), so this has
9574                // to be checked before the catch-all `Failed` below or the
9575                // one shape this exists for is lost with the rest.
9576                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9577                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9578                Ok(o) => AgentOutcome::Failed(format!(
9579                    "process exited with no usable output ({})",
9580                    crate::notices::exit_status(o.exit_code)
9581                )),
9582                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9583                    AgentOutcome::ResourceBlocked(e.to_string())
9584                }
9585                Err(e) => AgentOutcome::Failed(e.to_string()),
9586            };
9587            (i, seat, out, interrupted)
9588        });
9589    }
9590    drop(spawned_tx);
9591    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9592    loop {
9593        let joined = tokio::select! {
9594            Some(key) = spawned_rx.recv() => {
9595                // The clock of this seat starts now, not when it queued.
9596                state.seat_spawned(&key);
9597                if let Err(e) = state.save() {
9598                    tracing::warn!("could not persist a seat's start: {e:#}");
9599                }
9600                continue;
9601            }
9602            joined = set.join_next() => joined,
9603        };
9604        let Some(joined) = joined else {
9605            break;
9606        };
9607        let (i, seat, out, interrupted) = match joined {
9608            Ok(v) => v,
9609            // No seat to clear: a panicked task never reported which one it
9610            // was. The defensive sweep below this loop is what stops that
9611            // seat's `active` entry from surviving forever.
9612            Err(e) => {
9613                tracing::error!("agent task panicked: {e}");
9614                continue;
9615            }
9616        };
9617        state.seat_finished(&seat.key);
9618        if interrupted
9619            && node == "implement"
9620            && let Some(label) = state
9621                .candidates
9622                .iter()
9623                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9624                .map(|candidate| candidate.label)
9625        {
9626            state.candidate_preservation.insert(
9627                label,
9628                "interrupted implementation awaiting persistence".into(),
9629            );
9630            state.interrupted_implementations.insert(label);
9631        }
9632        if let AgentOutcome::ResourceBlocked(why) = &out
9633            && state.resource_block.is_none()
9634        {
9635            state.resource_block = Some(crate::run::ResourceBlock {
9636                node: node.to_owned(),
9637                kind: if why.contains("disk pressure") {
9638                    "disk pressure"
9639                } else if why.contains("repository coordination") {
9640                    "repository coordination"
9641                } else {
9642                    "parallel slot"
9643                }
9644                .to_owned(),
9645                detail: why.clone(),
9646                at: Timestamp::now(),
9647            });
9648            let activity = if interrupted {
9649                "agent execution interrupted"
9650            } else {
9651                "no agent was started"
9652            };
9653            state.event("resource", format!("{node}: {activity} - {why}"));
9654        }
9655        record_jobs(state, node, round, &seat.key, &out);
9656        if let Err(e) = state.save() {
9657            tracing::warn!("could not persist a seat's completion: {e:#}");
9658        }
9659        if collected.len() <= i {
9660            collected.resize_with(i + 1, || None);
9661        }
9662        collected[i] = Some((i, seat, out));
9663    }
9664    // Belt-and-braces for the panic branch above: every seat this exact batch
9665    // started shares this `(node, attempt)` pair, and every seat that finished
9666    // normally already cleared itself, so anything left tagged with it here
9667    // can only be a panicked task's leftover. Cleared unconditionally rather
9668    // than left to read as still answering forever.
9669    if state
9670        .active
9671        .values()
9672        .any(|a| a.node == node && a.attempt == attempt)
9673    {
9674        state
9675            .active
9676            .retain(|_, a| !(a.node == node && a.attempt == attempt));
9677        if let Err(e) = state.save() {
9678            tracing::warn!("could not persist the end of a wave: {e:#}");
9679        }
9680    }
9681    // Whether or not the lease above was actually held, several worktrees
9682    // may just have built into the cache with nothing here able to name one
9683    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
9684    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
9685    // clean it might not have strictly needed; trusting a stale match would
9686    // cost it a wrong answer.
9687    if let Some(cache_dir) = cache
9688        && jobs_had_a_writer
9689    {
9690        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9691    }
9692    if let Some(guard) = cache_guard {
9693        guard.release();
9694    }
9695    collected.into_iter().flatten().collect()
9696}
9697
9698/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
9699/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
9700/// this is data collection, not the fix-specific completion contract in
9701/// [`Runner::continue_fix_report`], and applies regardless of which node
9702/// asked.
9703///
9704/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
9705/// evidence from; `Failed` does not, and correctly contributes nothing — a
9706/// timeout or crash is not itself evidence about a command the seat may have
9707/// started.
9708fn record_jobs(
9709    state: &mut RunState,
9710    node: &str,
9711    round: Option<usize>,
9712    seat: &str,
9713    out: &AgentOutcome,
9714) {
9715    let commands: &[agent::CommandEvidence] = match out {
9716        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9717        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9718    };
9719    let checked_at = Timestamp::now();
9720    for c in commands {
9721        state.jobs.push(JobRecord {
9722            node: node.to_owned(),
9723            round,
9724            seat: seat.to_owned(),
9725            id: c.id.clone(),
9726            description: c.description.clone(),
9727            checked_at,
9728            status: match c.exit_code {
9729                Some(0) => JobStatus::Completed,
9730                Some(_) => JobStatus::Failed,
9731                None => JobStatus::Unknown,
9732            },
9733            exit_code: c.exit_code,
9734            result_summary: c.result_summary.clone(),
9735            source: c.source.clone(),
9736        });
9737    }
9738}
9739
9740/// Is a review round clean, given how many reviewer seats answered against
9741/// how many the round expected?
9742///
9743/// A seat that never answered (timeout, crash, unparsable output) is not a
9744/// seat that read the patch and found nothing — treating it as such is
9745/// exactly the bug this function exists to close. Under the default `block`
9746/// policy a missing seat can never be clean; `warn` still requires the seats
9747/// that *did* answer to have found nothing blocking and verification to be
9748/// green.
9749///
9750/// `quota_missing` narrows that `block` default for exactly one cause of
9751/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
9752/// a session limit lifts by the very next round buys nothing — the seat is
9753/// asked again with the same quota — so once every missing seat is accounted
9754/// for by a quota loss (and at least one seat *did* answer, so a decision has
9755/// something to rest on) the round is decided on the panel that could answer,
9756/// same as `warn` would. A panel that lost every seat to quota is not
9757/// decided here: `answered == 0` falls through to the existing `block`
9758/// fallback so a fully collapsed panel still waits rather than landing on no
9759/// review at all.
9760fn round_is_clean(
9761    blocking: usize,
9762    e2e_ok: bool,
9763    answered: usize,
9764    expected: usize,
9765    quota_missing: usize,
9766    policy: IncompleteReviewPolicy,
9767) -> bool {
9768    if blocking != 0 || !e2e_ok {
9769        return false;
9770    }
9771    if answered == expected || policy == IncompleteReviewPolicy::Warn {
9772        return true;
9773    }
9774    answered > 0 && expected - answered <= quota_missing
9775}
9776
9777/// The review loop's own conclusion, derived entirely from its persisted
9778/// round records and the round budget that produced them — never from
9779/// `status`, so a reentry (or `gate`/`merge` reading it independently)
9780/// recomputes the identical answer regardless of what an earlier node in the
9781/// same walk, or a previous walk, did to `status`.
9782///
9783/// `None` while more rounds remain to try, including when review never ran
9784/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
9785/// gone clean, or the budget is spent, or the tree has stopped moving (see
9786/// [`STAGNANT_LIMIT`]), the answer is one of two things:
9787///
9788/// - An incomplete panel that raised nothing is missing input, not a
9789///   verified tree — never a hand-off candidate, whatever verification said
9790///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
9791/// - Otherwise, green e2e on the last round hands off (see
9792///   [`Runner::stop_reviewing`]); red e2e blocks.
9793///
9794/// A last round whose own verification is still `ResourceBlocked` — magi
9795/// itself never got a command to run, not evidence the patch is broken —
9796/// is neither: this returns `None` for it too, the same as "more rounds
9797/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
9798/// handling of that shape) instead of this cheap recomputation guessing a
9799/// verdict a real attempt never produced.
9800pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9801    if max_rounds == 0
9802        || reviews.last().is_some_and(|r| {
9803            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
9804        })
9805    {
9806        return Some(RunStatus::Gating);
9807    }
9808    let last = reviews.last()?;
9809    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9810    if reviews.len() < max_rounds && !stagnant {
9811        return None;
9812    }
9813    if last.incomplete() && last.blocking == 0 {
9814        return Some(RunStatus::Blocked);
9815    }
9816    if last.e2e_status() == E2eStatus::ResourceBlocked
9817        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
9818    {
9819        return None;
9820    }
9821    Some(if last.e2e.iter().all(CommandOutcome::ok) {
9822        RunStatus::Gating
9823    } else {
9824        RunStatus::Blocked
9825    })
9826}
9827
9828/// How long a re-ask may take, given the budget the first attempt had.
9829///
9830/// A `nudged` retry is a request to restate an answer the seat has already
9831/// worked out: it carries no new work, so it does not deserve the original
9832/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
9833/// 133 seconds while a third sat for over ten minutes on a resumed session
9834/// holding 410 KB of prior output - and because the retry had inherited the
9835/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
9836/// judging round whose other seats were long finished.
9837///
9838/// A quarter of the budget, with a floor so that a deliberately short timeout
9839/// does not collapse to nothing. A retry that re-sends the whole prompt
9840/// (because the seat kept no context) is the original job again, and keeps the
9841/// original budget.
9842pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9843    if nudged {
9844        (full / 4).max(Duration::from_secs(120)).min(full)
9845    } else {
9846        full
9847    }
9848}
9849
9850/// Run a wave and parse each reply, re-asking the seats whose reply was
9851/// unusable.
9852///
9853/// The re-ask is a nudge rather than the whole prompt again when the seat still
9854/// holds its conversation, which is the difference between a cheap retry and
9855/// paying for the entire candidate set twice.
9856///
9857/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
9858/// successor in `roster` is handed to it instead of being re-asked: the
9859/// handover is the retry. A seat with no successor left (a single-agent
9860/// roster, the roster's tail) is nudged as before, up to `retries` times. So
9861/// the asks to one seat in one node number at most
9862/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
9863/// is asked once (a dropped stream is nudged first), and only the last agent
9864/// of the chain gets the `retries` same-agent nudges. Each roster agent is
9865/// tried at most once per seat, walking forward from the seat's own position and never wrapping
9866/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
9867/// other failure stops the chain when the previous agent failed the same way
9868/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
9869/// [`has_context`] is false and the job's own full prompt and full budget are
9870/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
9871/// intermediate ones are not losses) and is returned as a failure like any
9872/// other absent seat — the caller decides whether the panel still has a
9873/// quorum. An empty `roster` disables handover: failures are nudged as they
9874/// always were, and a quota is simply lost. A reply that fails to parse or
9875/// validate is the prompt's doing and is only ever nudged, never handed over.
9876///
9877/// The returned [`SeatState`] names the agent that answered (or tried last).
9878#[allow(clippy::too_many_arguments)]
9879async fn ask_json_wave<T>(
9880    jobs: Vec<SeatJob>,
9881    sem: Arc<Semaphore>,
9882    retries: usize,
9883    roster: &[AgentSpec],
9884    ctx: &WaveCtx<'_>,
9885    losses: &mut Vec<QuotaLoss>,
9886    state: &mut RunState,
9887    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9888) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9889where
9890    T: serde::de::DeserializeOwned + Send + 'static,
9891{
9892    ask_wave_with(
9893        jobs,
9894        sem,
9895        retries,
9896        roster,
9897        ctx,
9898        losses,
9899        state,
9900        &|text: &str| {
9901            let v = verdict::extract_json::<T>(text)?;
9902            validate(&v)?;
9903            Ok(v)
9904        },
9905    )
9906    .await
9907}
9908
9909/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
9910/// a node whose answer is prose (deliberation) shares the same handover,
9911/// failure classification, quota bookkeeping and bounds instead of a copy.
9912///
9913/// A seat handed to another roster agent is sent the job's `handover` prompt
9914/// (when it has one) rather than `prompt`: the new agent has no session, so a
9915/// resume-style prompt would be incomplete. That holds for the handover ask
9916/// and for every nudge to that agent whose `has_context` is false.
9917#[allow(clippy::too_many_arguments)]
9918async fn ask_wave_with<T>(
9919    jobs: Vec<SeatJob>,
9920    sem: Arc<Semaphore>,
9921    retries: usize,
9922    roster: &[AgentSpec],
9923    ctx: &WaveCtx<'_>,
9924    losses: &mut Vec<QuotaLoss>,
9925    state: &mut RunState,
9926    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9927) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9928where
9929    T: Send + 'static,
9930{
9931    let n = jobs.len();
9932    let originals: Vec<SeatJob> = jobs;
9933    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9934    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9935    // Nudges each seat's *current* agent has taken — 0 for a first-ask
9936    // answer, N once it has gone through N nudges. Read back once this
9937    // returns, so a caller building a history record (`ReviewRecord`) can
9938    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
9939    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
9940    // field's own doc.
9941    let mut nudges: Vec<usize> = vec![0; n];
9942    // The agent now occupying each seat, the ids it has already been through,
9943    // where in the roster the walk began, the class of the last failure, and
9944    // the stem word of a handover not yet asked (full prompt, full budget).
9945    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9946    let mut tried: Vec<BTreeSet<String>> = specs
9947        .iter()
9948        .map(|s| BTreeSet::from([s.id.clone()]))
9949        .collect();
9950    let starts: Vec<usize> = specs
9951        .iter()
9952        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9953        .collect();
9954    let carry = ctx.carry_seats && !roster.is_empty();
9955    // Carried across rounds: ids that failed the seat earlier, and how the
9956    // last failure went (so a repeat of it is not handed over again).
9957    let carried: Vec<BTreeSet<String>> = originals
9958        .iter()
9959        .map(|j| {
9960            state
9961                .seat_history
9962                .get(&j.seat.key)
9963                .filter(|_| carry)
9964                .map(|h| h.failed.clone())
9965                .unwrap_or_default()
9966        })
9967        .collect();
9968    let mut prev: Vec<Option<FailClass>> = originals
9969        .iter()
9970        .map(|j| {
9971            state
9972                .seat_history
9973                .get(&j.seat.key)
9974                .filter(|_| carry)
9975                .and_then(|h| h.last_fail.clone())
9976        })
9977        .collect();
9978    let next_agent =
9979        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9980            let others: BTreeSet<String> = specs
9981                .iter()
9982                .enumerate()
9983                .filter(|(j, _)| *j != i)
9984                .map(|(_, s)| s.id.clone())
9985                .collect();
9986            pick_successor(
9987                roster,
9988                starts[i],
9989                tried,
9990                carry.then(|| &carried[i]),
9991                &others,
9992            )
9993            .cloned()
9994        };
9995    let mut fresh: Vec<Option<String>> = vec![None; n];
9996    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9997    let mut pending: Vec<usize> = (0..n).collect();
9998
9999    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
10000    // asks: an agent with a successor is asked once and handed over, and only
10001    // a seat with no successor spends `retries` nudges on the same agent. This
10002    // only guarantees the loop's own termination whatever those say.
10003    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10004    for round in 0..max_rounds {
10005        if pending.is_empty() {
10006            break;
10007        }
10008        let mut batch = Vec::with_capacity(pending.len());
10009        let mut renudged: Vec<&str> = Vec::new();
10010        for &i in &pending {
10011            let src = &originals[i];
10012            // A seat now held by another agent than the job named has no
10013            // session of its own: it gets the full-context prompt whenever
10014            // it is asked in full (the handover ask, a nudge it cannot
10015            // resume).
10016            let full: &str = match &src.handover {
10017                Some(h) if specs[i].id != src.spec.id => h,
10018                _ => &src.prompt,
10019            };
10020            // The prompt and the budget are one decision: a nudge restates
10021            // finished work, a re-sent prompt redoes it.
10022            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10023                (
10024                    full.to_owned(),
10025                    src.timeout,
10026                    format!("{}-{word}-{}", src.stem, specs[i].id),
10027                )
10028            } else if nudges[i] == 0 {
10029                (full.to_owned(), src.timeout, src.stem.clone())
10030            } else {
10031                renudged.push(src.seat.key.as_str());
10032                let why = done[i]
10033                    .as_ref()
10034                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
10035                    .unwrap_or_else(|| "no parsable answer".to_owned());
10036                let nudge = prompt::nudge(&why);
10037                let nudged = has_context(&specs[i], &seats[i], src.sessions);
10038                let prompt = if nudged {
10039                    nudge
10040                } else {
10041                    format!("{full}\n\n---\n\n{nudge}")
10042                };
10043                (
10044                    prompt,
10045                    retry_budget(src.timeout, nudged),
10046                    format!("{}-retry{}", src.stem, nudges[i]),
10047                )
10048            };
10049            batch.push(SeatJob {
10050                spec: specs[i].clone(),
10051                seat: seats[i].clone(),
10052                cwd: src.cwd.clone(),
10053                prompt,
10054                timeout,
10055                allow_write: src.allow_write,
10056                sessions: src.sessions,
10057                artifacts: src.artifacts.clone(),
10058                stem,
10059                handover: None,
10060            });
10061        }
10062
10063        if !renudged.is_empty() {
10064            state.event(
10065                ctx.node,
10066                format!("retry {round}: re-asking {}", renudged.join(", ")),
10067            );
10068        }
10069        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10070        let mut still = Vec::new();
10071        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10072            seats[i] = seat;
10073            // No agent was asked: nothing to hand over, nudge, remember or
10074            // charge. The caller sees `resource_block` and stops the node.
10075            if let AgentOutcome::ResourceBlocked(why) = &out {
10076                done[i] = Some(Err(anyhow::anyhow!("{why}")));
10077                continue;
10078            }
10079            let class = FailClass::of(&out);
10080            // A dropped stream is nudged first (the conversation is still
10081            // there to pick up); only a seat whose nudges are spent hands over.
10082            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10083                && nudges[i] < retries
10084                && !roster.is_empty();
10085            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10086                let next = should_hand_over(prev[i].as_ref(), &cur)
10087                    .then(|| next_agent(i, &tried[i], &specs))
10088                    .flatten();
10089                if carry {
10090                    let h = state
10091                        .seat_history
10092                        .entry(originals[i].seat.key.clone())
10093                        .or_default();
10094                    h.failed.insert(specs[i].id.clone());
10095                    h.last_fail = Some(cur.clone());
10096                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10097                        h.last_ok = None;
10098                    }
10099                    // Saved before the next agent is asked, so a restart in
10100                    // between does not forget who failed.
10101                    if let Err(e) = state.save() {
10102                        tracing::warn!("could not persist a seat's failure history: {e:#}");
10103                    }
10104                }
10105                if let Some(next) = next {
10106                    record_handover(
10107                        state,
10108                        ctx.node,
10109                        &originals[i].seat.key,
10110                        &specs[i].id,
10111                        &next.id,
10112                        &cur,
10113                        &fail_reason(&out),
10114                    );
10115                    tried[i].insert(next.id.clone());
10116                    prev[i] = Some(cur.clone());
10117                    seats[i] =
10118                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10119                    specs[i] = next;
10120                    fresh[i] = Some(cur.stem_word().to_owned());
10121                    nudges[i] = 0;
10122                    done[i] = Some(Err(anyhow::anyhow!(
10123                        "handed over after: {}",
10124                        fail_reason(&out)
10125                    )));
10126                    still.push(i);
10127                    continue;
10128                }
10129            }
10130            if carry
10131                && !nudge_first
10132                && let Some(cur) = class.clone()
10133            {
10134                // The chain ended here (no successor, or a repeated failure).
10135                let h = state
10136                    .seat_history
10137                    .entry(originals[i].seat.key.clone())
10138                    .or_default();
10139                h.failed.insert(specs[i].id.clone());
10140                h.last_fail = Some(cur);
10141            }
10142            let parsed = match out {
10143                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10144                AgentOutcome::Quota(o) => {
10145                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10146                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10147                }
10148                // Not a parseable answer: the nudge loop re-asks it, which is
10149                // exactly what a dropped stream needs. Just don't hand its raw
10150                // error JSON to `extract_json`.
10151                AgentOutcome::Dropped(o) => {
10152                    let why = o
10153                        .dropped
10154                        .as_ref()
10155                        .map(|d| d.why.as_str())
10156                        .unwrap_or("the CLI ended the stream without delivering its answer");
10157                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10158                }
10159                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10160                    Err(anyhow::anyhow!(e))
10161                }
10162            };
10163            let quota = class == Some(FailClass::Quota);
10164            let failed = parsed.is_err();
10165            done[i] = Some(parsed);
10166            if carry && !failed {
10167                let h = state
10168                    .seat_history
10169                    .entry(originals[i].seat.key.clone())
10170                    .or_default();
10171                h.failed.remove(&specs[i].id);
10172                h.last_ok = Some(specs[i].id.clone());
10173                h.last_fail = None;
10174            }
10175            // Do not re-ask a rate-limited seat (quota) — a retry is known to
10176            // fail the same way; and never re-ask a seat that already parsed.
10177            // A failed agent that still has a successor is not re-asked
10178            // either: the handover was its remedy and has just been refused
10179            // (the chain stops on a repeated failure class). A seat with no
10180            // successor left (a single-agent roster, the roster's tail, or an
10181            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10182            let agent_failure = class.is_some()
10183                && !nudge_first
10184                && !roster.is_empty()
10185                && next_agent(i, &tried[i], &specs).is_some();
10186            if failed && !quota && !agent_failure && nudges[i] < retries {
10187                nudges[i] += 1;
10188                still.push(i);
10189            }
10190        }
10191        pending = still;
10192    }
10193
10194    // One loss per seat whose chain ended on a quota: the intermediate ones
10195    // were absorbed by a handover and are not losses.
10196    for (i, q) in last_quota.into_iter().enumerate() {
10197        if let Some(reset) = q {
10198            losses.push(QuotaLoss {
10199                seat: originals[i].seat.key.clone(),
10200                node: ctx.node.to_owned(),
10201                at: Timestamp::now(),
10202                reset,
10203            });
10204        }
10205    }
10206
10207    seats
10208        .into_iter()
10209        .zip(done)
10210        .zip(nudges)
10211        .map(|((seat, res), attempts)| {
10212            (
10213                seat,
10214                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10215                attempts,
10216            )
10217        })
10218        .collect()
10219}
10220
10221/// Did the wave end without asking any agent (see [`block_wave`])?
10222fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10223    results
10224        .iter()
10225        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10226}
10227
10228/// Settle a wave that never spawned anything: every job answers
10229/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10230/// else about the seat (session, history, quota) is touched.
10231fn block_wave(
10232    jobs: Vec<SeatJob>,
10233    state: &mut RunState,
10234    node: &str,
10235    attempt: usize,
10236    why: &str,
10237) -> Vec<(usize, SeatState, AgentOutcome)> {
10238    state.resource_wait = None;
10239    state
10240        .active
10241        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10242    if let Err(e) = state.save() {
10243        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10244    }
10245    jobs.into_iter()
10246        .enumerate()
10247        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10248        .collect()
10249}
10250
10251/// Acquire the shared build cache's lease, waiting out contention within
10252/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10253/// unbounded wait is never acceptable).
10254///
10255/// A first, non-blocking check happens before ever waiting; if it finds the
10256/// lease busy, that fact is logged as a `verify` event *and* flushed with
10257/// [`RunState::save`] immediately — not only once the wait finally succeeds
10258/// or gives up — so a `magi show` run by a different process while this one
10259/// is still waiting reads a `run.json` that says so, rather than whatever it
10260/// looked like before the wait started. The same applies to the terminal
10261/// failure: logged and saved before this returns `Err`, so a caller that
10262/// could not get the lease at all still leaves a legible record of why.
10263async fn acquire_cache_lease(
10264    state: &mut RunState,
10265    cache_dir: &Path,
10266    owner: &crate::cache::Owner,
10267    budget: Duration,
10268    context: &str,
10269    cancel: &(dyn Fn() -> bool + Sync),
10270) -> Result<Option<crate::cache::Guard>> {
10271    let home = crate::run::home();
10272    let started = Instant::now();
10273    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10274        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10275        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10276        Err(e) => {
10277            state.event(
10278                "verify",
10279                format!("{context}: could not check the shared build cache: {e:#}"),
10280            );
10281            if let Err(e2) = state.save() {
10282                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10283            }
10284            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10285        }
10286    };
10287    state.event(
10288        "verify",
10289        format!(
10290            "{context}: waiting for the shared build cache at {} ({})",
10291            cache_dir.display(),
10292            busy.describe()
10293        ),
10294    );
10295    // Seats of this node are waiting, not running: say so, so the countdown a
10296    // reader sees only ever measures an agent that has been spawned.
10297    state.resource_wait = Some(crate::run::ResourceWait {
10298        node: context.to_owned(),
10299        kind: "shared build cache".to_owned(),
10300        holder: busy.describe(),
10301        since: Timestamp::now(),
10302        limit_secs: budget.as_secs(),
10303    });
10304    if let Err(e) = state.save() {
10305        tracing::warn!("could not persist a cache wait: {e:#}");
10306    }
10307    let remaining = budget.saturating_sub(started.elapsed());
10308    let waited = crate::cache::wait_for_unless(
10309        &home,
10310        cache_dir,
10311        owner,
10312        remaining,
10313        Duration::from_secs(5),
10314        cancel,
10315    )
10316    .await;
10317    state.resource_wait = None;
10318    match waited {
10319        Ok(g) => Ok(g),
10320        Err(e) => {
10321            state.event("verify", format!("{context}: {e:#}"));
10322            if let Err(e2) = state.save() {
10323                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10324            }
10325            Err(e)
10326        }
10327    }
10328}
10329
10330/// The lease state could not be read at all (unreadable lease file, a
10331/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10332/// known to be contended, so a wave degrades to running without a shared cache
10333/// instead of stopping on a hold that [`crate::triage`] could never release.
10334#[derive(Debug)]
10335struct CacheCheckError(String);
10336
10337impl std::fmt::Display for CacheCheckError {
10338    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10339        f.write_str(&self.0)
10340    }
10341}
10342
10343impl std::error::Error for CacheCheckError {}
10344
10345/// Run `body` — a verify command batch — while holding the shared build
10346/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10347/// never interleave with another borrower's build against the same
10348/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10349/// timeout, or a human's own `magi review`. See the `cache` module doc for
10350/// why this matters more than Cargo's own per-target locking covers — two
10351/// *different* worktrees building the same package name/version into one
10352/// cache directory is a staleness bug, not a lock contention one.
10353///
10354/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10355/// agent waves do. Repository scheduling also has its own bounded,
10356/// cancellable resource-wait policy. Contention must not consume the
10357/// verification budget: `body` receives the full `budget` after both leases
10358/// are acquired and the cache is fresh. Without a shared cache, only
10359/// repository coordination is acquired before handing over the full budget.
10360///
10361/// A lease that cannot be acquired within the resource wait is reported as a single
10362/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10363/// verification — the same shape a spawn failure already takes in
10364/// [`run_commands`], so a caller need not special-case it.
10365#[allow(clippy::too_many_arguments)]
10366async fn with_cache_lease<'s, F, Fut>(
10367    state: &'s mut RunState,
10368    cache_dir: Option<&Path>,
10369    node: &str,
10370    seat: &str,
10371    worktree: &Path,
10372    head: &str,
10373    budget: Duration,
10374    context: &str,
10375    wait: &crate::repo_lock::WaitPolicy,
10376    body: F,
10377) -> Result<(Vec<CommandOutcome>, bool)>
10378where
10379    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10380    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10381{
10382    // Match editing waves: cache first, repository second. In particular,
10383    // pre_gate must never hold the repository while waiting for a wave's cache.
10384    let started = Instant::now();
10385    let home = crate::run::home();
10386    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10387    let guard = if let Some(cache_dir) = cache_dir {
10388        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10389        match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10390            wait.cancelled()
10391        })
10392        .await
10393        .and_then(|g| {
10394            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10395        }) {
10396            Ok(g) => Some(g),
10397            Err(e) => {
10398                return Ok((
10399                    vec![CommandOutcome {
10400                        command: "(waiting for the shared build cache)".into(),
10401                        code: None,
10402                        output_tail: e.to_string(),
10403                        duration_ms: started.elapsed().as_millis() as u64,
10404                        resource_blocked: true,
10405                    }],
10406                    false,
10407                ));
10408            }
10409        }
10410    } else {
10411        None
10412    };
10413    let _coordination = coordinate_operation(state, worktree, wait).await?;
10414    if let Some(cache_dir) = cache_dir {
10415        let identity = crate::cache::Identity::new(worktree, head);
10416        if let Err(e) = crate::cache::ensure_fresh(
10417            &home,
10418            cache_dir,
10419            &identity,
10420            &state.config.verify.cargo_manifests(),
10421        ) {
10422            state.event(
10423                "verify",
10424                format!(
10425                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10426                    worktree.display(),
10427                    short(head)
10428                ),
10429            );
10430            if let Some(guard) = guard {
10431                guard.release();
10432            }
10433            return Ok((
10434                vec![CommandOutcome {
10435                    command: "(confirming the shared build cache is fresh)".into(),
10436                    code: None,
10437                    output_tail: e.to_string(),
10438                    duration_ms: started.elapsed().as_millis() as u64,
10439                    resource_blocked: true,
10440                }],
10441                false,
10442            ));
10443        }
10444    }
10445    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10446    if !timed_out_pids.is_empty() {
10447        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10448    }
10449    // Drop repository coordination before releasing the cache, reversing the
10450    // acquisition order. A cancelled/error future also drops both guards.
10451    drop(_coordination);
10452    if let Some(guard) = guard {
10453        guard.release();
10454    }
10455    Ok((outcomes, retried))
10456}
10457
10458/// Poll `pids` — commands [`run_commands`] reports as still running when its
10459/// own timeout elapsed — until every one is confirmed gone, or
10460/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10461///
10462/// Real confirmation where confirmation is possible, not a substitute for
10463/// full process-tree observation: a grandchild the timed-out process spawned
10464/// and that survives independently of it is invisible to a pid check the
10465/// same way it always was, and continuing to observe and collect *that*
10466/// stays a different piece of work with its own owner. This only narrows a
10467/// fixed blind wait into an actual check of the pids this process does know
10468/// about.
10469async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10470    wait_for_pids_with(
10471        pids,
10472        crate::proc::pid_alive,
10473        LEASE_RELEASE_POLL,
10474        LEASE_RELEASE_MAX_WAIT,
10475    )
10476    .await;
10477}
10478
10479/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10480/// interval and ceiling supplied by the caller, so the polling *logic* -
10481/// returns as soon as every pid reports dead, gives up at the ceiling
10482/// otherwise - is testable on millisecond durations without asking the real
10483/// OS about a pid at all.
10484async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10485    pids: &[u32],
10486    alive: F,
10487    poll: Duration,
10488    max_wait: Duration,
10489) {
10490    let deadline = Instant::now() + max_wait;
10491    loop {
10492        if pids.iter().all(|&pid| !alive(pid)) {
10493            return;
10494        }
10495        if Instant::now() >= deadline {
10496            return;
10497        }
10498        tokio::time::sleep(poll).await;
10499    }
10500}
10501
10502/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10503/// admission that it could not even get a verify command to run, as opposed
10504/// to evidence the command actually produced? A caller that would otherwise
10505/// read a resource-blocked outcome as a red command must check this first:
10506/// see [`Runner::gate`], which retries rather than records `Blocked` when
10507/// this is true.
10508fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10509    outcomes.iter().any(|o| o.resource_blocked)
10510}
10511
10512/// What [`Runner::gate_fix_round`] decided.
10513pub(crate) enum GateFix {
10514    /// The tree changed and `verify.e2e` is still green: run the gate again.
10515    Retry,
10516    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10517    /// last failure stands and the run ends blocked.
10518    Stop,
10519    /// `verify.e2e` could not run after the fix (magi's own contention):
10520    /// decide nothing now, a later reentry retries.
10521    Defer,
10522}
10523
10524/// Is every red command in `outcomes` an ordinary failure the code could
10525/// explain: it ran, exited non-zero, and said something?
10526///
10527/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10528/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10529/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10530/// what the command is stays the gate's business.
10531fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10532    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10533    red.peek().is_some()
10534        && red.all(|o| {
10535            !o.resource_blocked
10536                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10537                && !o.output_tail.trim().is_empty()
10538        })
10539}
10540
10541/// Describe one verify command's outcome for the event log, distinguishing a
10542/// build/link failure — the toolchain never produced a binary to run — from
10543/// an actual test failure, since only the latter is a verdict on the patch.
10544fn e2e_outcome_label(o: &CommandOutcome) -> String {
10545    if o.ok() {
10546        return "pass".to_owned();
10547    }
10548    let reason = if o.build_failed() {
10549        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10550    } else {
10551        format!("FAIL ({:?})", o.code)
10552    };
10553    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10554}
10555
10556/// Run `verify.e2e`, retrying once if the first attempt could not build or
10557/// link — a build/link failure is frequently a race against a shared
10558/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
10559/// `verify` event per command, tagged with `context` (normally `"round N"`)
10560/// so the two call sites that need this — the ordinary per-round leg in
10561/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
10562/// it will ever call a round green — read identically in the event log.
10563async fn run_e2e_with_retry(
10564    state: &mut RunState,
10565    shell: &[String],
10566    commands: &[String],
10567    worktree: &Path,
10568    timeout: Duration,
10569    context: &str,
10570) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10571    let (mut e2e, mut timed_out_pids) = run_commands(
10572        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10573    )
10574    .await;
10575    for o in &e2e {
10576        state.event(
10577            "verify",
10578            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10579        );
10580    }
10581    // A build/link failure is not a verdict on the patch — it is frequently a
10582    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
10583    // one retry before letting a red like that decide the round.
10584    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10585    if verify_retried {
10586        state.event(
10587            "verify",
10588            format!(
10589                "{context}: verify could not build/link, not a test result — retrying once \
10590                 before concluding"
10591            ),
10592        );
10593        let retried = run_commands(
10594            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10595        )
10596        .await;
10597        e2e = retried.0;
10598        // Both attempts' timeouts matter, not just the last one: the first
10599        // attempt's descendants may still be alive alongside the retry's.
10600        timed_out_pids.extend(retried.1);
10601        for o in &e2e {
10602            state.event(
10603                "verify",
10604                format!(
10605                    "{context}: retry `{}` -> {}",
10606                    o.command,
10607                    e2e_outcome_label(o)
10608                ),
10609            );
10610        }
10611    }
10612    (e2e, verify_retried, timed_out_pids)
10613}
10614
10615/// Run configured shell commands in `cwd`, in order. The second element is
10616/// the pid of every command that hit `timeout` and was still running when
10617/// this stopped waiting on it (best-effort: `None` when the platform did not
10618/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
10619/// that releases a shared resource afterward needs to know.
10620///
10621/// Records `task` into [`RunState::active`] at every command boundary
10622/// (`RunState::task_command`) and clears it once the whole list has run
10623/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
10624/// for minutes with no seat and no output of its own to show for it (see
10625/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
10626/// apart from "not yet run" without this), and this is the only place that
10627/// knows which command is running right now and how many are left. Three
10628/// saves per command — start, not per second — matching the same "only at a
10629/// boundary" rule [`wave`] already follows for seats.
10630#[allow(clippy::too_many_arguments)]
10631async fn run_commands(
10632    state: &mut RunState,
10633    node: &str,
10634    task: &str,
10635    attempt: usize,
10636    shell: &[String],
10637    commands: &[String],
10638    cwd: &Path,
10639    timeout: Duration,
10640) -> (Vec<CommandOutcome>, Vec<u32>) {
10641    if commands.is_empty() {
10642        // Nothing to mark as running and nothing to clear — an empty list
10643        // means "not configured", and touching `active` (or the disk) over
10644        // that would be a write for every round of a repo with no
10645        // `verify.e2e` / `verify.gate` commands at all.
10646        return (Vec::new(), Vec::new());
10647    }
10648    let mut out = Vec::new();
10649    let mut timed_out_pids = Vec::new();
10650    let total = commands.len();
10651    for (idx, command) in commands.iter().enumerate() {
10652        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10653        if let Err(e) = state.save() {
10654            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10655        }
10656        let started = Instant::now();
10657        let mut cmd = tokio::process::Command::new(&shell[0]);
10658        cmd.quiet();
10659        cmd.args(&shell[1..])
10660            .arg(command)
10661            .current_dir(cwd)
10662            .stdin(std::process::Stdio::null())
10663            .stdout(std::process::Stdio::piped())
10664            .stderr(std::process::Stdio::piped())
10665            .kill_on_drop(true);
10666        let mut outputs = Vec::new();
10667        let protected = (|| -> Result<()> {
10668            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10669                outputs.push(guard);
10670            }
10671            if let Some(cache) = state.config.cache_dir() {
10672                outputs.push(crate::retention::OutputLease::cache(
10673                    &crate::run::home(),
10674                    &cache,
10675                    false,
10676                )?);
10677            }
10678            Ok(())
10679        })();
10680        if let Err(e) = protected {
10681            out.push(CommandOutcome {
10682                command: command.clone(),
10683                code: None,
10684                output_tail: format!("output ownership unavailable: {e:#}"),
10685                duration_ms: 0,
10686                resource_blocked: true,
10687            });
10688            continue;
10689        }
10690        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10691        let (code, body) = match spawned {
10692            Ok(mut running) => {
10693                let pid = running.child().id();
10694                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10695                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10696                let mut paths = vec![cwd.to_owned()];
10697                if let Some(cache) = state.config.cache_dir() {
10698                    paths.push(cache);
10699                }
10700                let mut pressure = None;
10701                let status = tokio::select! {
10702                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10703                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10704                        pressure = Some(reason); None
10705                    }
10706                };
10707                let (code, failure) = match status {
10708                    Some(Ok(Ok(status))) => (
10709                        status.code(),
10710                        status
10711                            .code()
10712                            .is_none()
10713                            .then(|| format!("command terminated: {status}")),
10714                    ),
10715                    Some(Ok(Err(error))) => {
10716                        (None, Some(format!("failed waiting for command: {error}")))
10717                    }
10718                    Some(Err(_)) => {
10719                        // Leased cleanup confirms process-tree death itself.
10720                        // Unleased commands still need the caller's PID check
10721                        // after kill-on-drop, before releasing cache ownership.
10722                        if !running.coordinated()
10723                            && let Some(pid) = pid
10724                        {
10725                            timed_out_pids.push(pid);
10726                        }
10727                        (
10728                            None,
10729                            Some(format!("timed out after {}s", timeout.as_secs())),
10730                        )
10731                    }
10732                    None => (None, None),
10733                };
10734                running.finish().await;
10735                if let Some(reason) = pressure {
10736                    state.resource_block = Some(crate::run::ResourceBlock {
10737                        node: node.to_owned(),
10738                        kind: "disk pressure".to_owned(),
10739                        detail: reason.clone(),
10740                        at: Timestamp::now(),
10741                    });
10742                    out.push(CommandOutcome {
10743                        command: command.clone(),
10744                        code: None,
10745                        output_tail: reason,
10746                        duration_ms: started.elapsed().as_millis() as u64,
10747                        resource_blocked: true,
10748                    });
10749                    break;
10750                }
10751                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10752                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10753                if let Some(failure) = failure {
10754                    body.push('\n');
10755                    body.push_str(&failure);
10756                }
10757                (code, body)
10758            }
10759            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10760        };
10761        out.push(CommandOutcome {
10762            command: command.clone(),
10763            code,
10764            output_tail: tail(&body, OUTPUT_TAIL),
10765            duration_ms: started.elapsed().as_millis() as u64,
10766            resource_blocked: false,
10767        });
10768    }
10769    state.task_finished(task);
10770    if let Err(e) = state.save() {
10771        tracing::warn!("could not persist the end of {task}: {e:#}");
10772    }
10773    (out, timed_out_pids)
10774}
10775
10776/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
10777/// section (`report::run`) and in the `merge` event this node records — for
10778/// the operator to run by hand.
10779///
10780/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
10781/// branch whose ruleset forbids merge commits (GitHub's "must not contain
10782/// merge commits", or "require linear history") rejects the push a `--no-ff`
10783/// merge would produce, which is exactly the guidance this function replaces.
10784/// `message`'s first line becomes the squash commit's subject, matching the
10785/// note `report::run` prints alongside this command — see that function for
10786/// why an explicit subject is not optional there.
10787fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10788    let repo = repo.display();
10789    match style {
10790        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10791        MergeStyle::Squash => {
10792            // The subject sits inside double quotes, and a title an agent
10793            // wrote may carry the characters that break out of them.
10794            let subject = message
10795                .lines()
10796                .next()
10797                .unwrap_or(branch)
10798                .replace(['\\', '"', '$', '`'], "");
10799            format!(
10800                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10801            )
10802        }
10803        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10804    }
10805}
10806
10807/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
10808/// under the hood, rejects a `title` over 256 characters and the whole
10809/// command fails — no PR at all, for a run whose body was otherwise fine
10810/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
10811/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
10812/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
10813/// this bound. It is a margin, not a guarantee — a title packed
10814/// with multi-unit characters could still in principle land close to the
10815/// edge, but a real task title's occasional emoji or accented letter fits
10816/// comfortably inside it.
10817const PR_TITLE_MAX: usize = 240;
10818
10819/// A pull request title from the opening line of `text`, or `None` when that
10820/// line is not English (GitHub text is) or has no letters.
10821///
10822/// A line within `max` is kept as is. A longer one is cut at the end of its
10823/// first sentence when that falls inside `max`, else at a word boundary with a
10824/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
10825/// make every merely-truncated title look non-English). The language check
10826/// runs on the kept text before any mark is added, so only what GitHub will
10827/// show is judged: an English opening followed by non-ASCII far past the cut
10828/// still passes.
10829fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10830    let line = queue::first_line(text)?;
10831    let chars: Vec<char> = line.chars().collect();
10832    let (kept, mark) = if chars.len() <= max {
10833        (line.to_owned(), "")
10834    } else if let Some(end) = sentence_end(&chars, max) {
10835        (chars[..end].iter().collect::<String>(), "")
10836    } else {
10837        let room = max.saturating_sub(3);
10838        // Cut at the last space inside the room; when the char just past the
10839        // room is a space the room already ends on a word.
10840        let cut = if chars[room].is_whitespace() {
10841            room
10842        } else {
10843            chars[..room]
10844                .iter()
10845                .rposition(|c| c.is_whitespace())
10846                .unwrap_or(room)
10847        };
10848        let head: String = chars[..cut].iter().collect();
10849        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10850        (head.to_owned(), "...")
10851    };
10852    if kept.is_empty() {
10853        return None;
10854    }
10855    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10856        return None;
10857    }
10858    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10859        return None;
10860    }
10861    Some(format!("{kept}{mark}"))
10862}
10863
10864/// The char length of the first sentence of `chars` when it ends within `max`
10865/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
10866/// abbreviations so `e.g. foo` does not end a title early.
10867fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10868    const MIN: usize = 20;
10869    for i in MIN..max.min(chars.len()) {
10870        if !matches!(chars[i], '.' | '!' | '?') {
10871            continue;
10872        }
10873        let Some(&next) = chars.get(i + 1) else {
10874            continue;
10875        };
10876        if !next.is_whitespace() {
10877            continue;
10878        }
10879        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10880        if after.is_some_and(|c| c.is_ascii_lowercase()) {
10881            continue;
10882        }
10883        let word: String = chars[..i]
10884            .iter()
10885            .rev()
10886            .take_while(|c| !c.is_whitespace())
10887            .collect::<Vec<_>>()
10888            .into_iter()
10889            .rev()
10890            .collect();
10891        let word = word.to_ascii_lowercase();
10892        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10893            continue;
10894        }
10895        let end = chars[..i]
10896            .iter()
10897            .rposition(|c| !c.is_whitespace())
10898            .map_or(i, |p| p + 1);
10899        return Some(end);
10900    }
10901    None
10902}
10903
10904/// What `merge = "pr"` (and the merge commit of the other modes) says about a
10905/// change: a title and a body describing what was *implemented*, not the task
10906/// that asked for it. A task reads as a request; a reader of the merged
10907/// history wants the change.
10908struct PrMessage {
10909    title: String,
10910    body: String,
10911}
10912
10913impl PrMessage {
10914    /// Title, blank line, body. The first line is the squash/merge commit
10915    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
10916    /// has to stay one sensible line.
10917    fn commit_message(&self) -> String {
10918        format!("{}\n\n{}", self.title, self.body)
10919    }
10920}
10921
10922/// The text after a leading `TITLE:` (any case) on `line`.
10923fn title_marker(line: &str) -> Option<&str> {
10924    let line = line.trim();
10925    let head = line.get(..6)?;
10926    head.eq_ignore_ascii_case("title:")
10927        .then(|| line[6..].trim())
10928}
10929
10930/// The implementer's own one-line title: the `TITLE:` line the implement
10931/// prompt asks for at the top of its SUMMARY. Candidate commits are all
10932/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
10933/// source, and a title that says as much is refused here too.
10934fn summary_title(summary: &str) -> Option<String> {
10935    let first = summary.lines().find(|l| !l.trim().is_empty())?;
10936    let raw = title_marker(first)?;
10937    if raw.is_empty() {
10938        return None;
10939    }
10940    let title = queue::title_from(raw, PR_TITLE_MAX);
10941    let lower = title.to_ascii_lowercase();
10942    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10943        return None;
10944    }
10945    Some(title)
10946}
10947
10948/// How `open_review`'s instruction begins; see [`landing_title`].
10949const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10950
10951/// Marker `open_review` gives a candidate that nothing in the roster wrote.
10952pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10953
10954/// Does this run review work that already existed, rather than implement a
10955/// task? Runs recorded before `reviewed_commits` existed carry only the
10956/// candidate marker.
10957pub(crate) fn is_review_run(state: &RunState) -> bool {
10958    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10959}
10960
10961/// The title of a review-only run: the subject of the oldest commit under
10962/// review. Later commits are usually fixups, and `instruction` is the review
10963/// prompt, which says nothing about the change. GitHub text is English, so a
10964/// non-ASCII or blank subject yields `None` and the caller's neutral title.
10965fn review_title(state: &RunState) -> Option<String> {
10966    english_subject(
10967        state.reviewed_commits.as_ref()?.first()?,
10968        state.config.graph.github_english(),
10969    )
10970}
10971
10972/// `raw` as a pull request title, or `None` when it is blank, not English
10973/// (GitHub text is), or one of magi's own candidate commit subjects.
10974fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10975    let raw = raw.trim();
10976    if raw.is_empty() {
10977        return None;
10978    }
10979    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10980        return None;
10981    }
10982    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10983        return None;
10984    }
10985    let title = queue::title_from(raw, PR_TITLE_MAX);
10986    let lower = title.to_ascii_lowercase();
10987    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10988        return None;
10989    }
10990    Some(title)
10991}
10992
10993/// What a review-only run's branch says about itself, read at the moment the
10994/// pull request is opened.
10995#[derive(Debug, Clone, PartialEq, Eq)]
10996struct BranchFacts {
10997    /// `(subject, body)` of each commit, oldest first.
10998    commits: Vec<(String, String)>,
10999    /// Trimmed `git diff --stat`.
11000    stat: String,
11001}
11002
11003/// Longest diff stat shown: this many file lines plus the summary line.
11004const STAT_FILE_LINES: usize = 25;
11005/// Cap on the commit list, well inside GitHub's 65536-character body limit.
11006const COMMITS_MAX_CHARS: usize = 20_000;
11007
11008/// Read the commits and diff stat of `base..branch`. `None` when git cannot
11009/// say or finds nothing, so the caller falls back to what the run recorded.
11010async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11011    let commits = git::commit_log(repo, base, branch).await.ok()?;
11012    if commits.is_empty() {
11013        return None;
11014    }
11015    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11016    let lines: Vec<&str> = stat.lines().collect();
11017    let stat = if lines.len() > STAT_FILE_LINES + 1 {
11018        let omitted = lines.len() - 1 - STAT_FILE_LINES;
11019        let more = format!(" ... {omitted} more file(s)");
11020        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11021        kept.push(&more);
11022        kept.push(lines[lines.len() - 1]);
11023        kept.join("\n")
11024    } else {
11025        lines.join("\n")
11026    };
11027    Some(BranchFacts { commits, stat })
11028}
11029
11030/// Defang what would break the surrounding markdown: a closing `</details>`
11031/// and a code fence.
11032fn markdown_safe(text: &str) -> String {
11033    text.replace("</details>", "&lt;/details&gt;")
11034        .replace("\x60\x60\x60", "~~~")
11035}
11036
11037fn neutral_title(state: &RunState, winner: char) -> String {
11038    format!(
11039        "chore: land candidate {} of run {}",
11040        winner.to_ascii_uppercase(),
11041        state.id
11042    )
11043}
11044
11045/// Reuse the graph's bounded fixer without starting another graph or driver.
11046/// Copy the working state back even on errors so spent rounds remain visible.
11047pub(crate) async fn repair_landing_gate(
11048    state: &mut RunState,
11049    winner: &Candidate,
11050    outcomes: &[CommandOutcome],
11051) -> Result<GateFix> {
11052    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11053        return Ok(GateFix::Stop);
11054    }
11055    let mut runner = Runner {
11056        roles: state.config.resolve_roles()?,
11057        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11058        state: state.clone(),
11059        pause: Pause::new(),
11060        interrupt: Pause::new(),
11061    };
11062    let result = runner.gate_fix_round(winner, outcomes).await;
11063    *state = runner.state;
11064    result
11065}
11066
11067/// Recheck a repair's e2e requirement across failures, resource deferrals and
11068/// process interruptions. A successful result only applies to its exact SHA.
11069pub(crate) async fn check_gate_repair_e2e(
11070    state: &mut RunState,
11071    winner: &Candidate,
11072    wait: &crate::repo_lock::WaitPolicy,
11073) -> Result<GateFix> {
11074    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11075        return Ok(GateFix::Retry);
11076    }
11077    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11078    if state
11079        .gate_repair_e2e
11080        .as_ref()
11081        .is_some_and(|r| r.head == head && r.passed)
11082    {
11083        return Ok(GateFix::Retry);
11084    }
11085    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11086        head: head.clone(),
11087        passed: false,
11088        outcomes: Vec::new(),
11089    });
11090    state.save()?;
11091    let commands = state.config.verify.e2e.clone();
11092    let outcomes = if commands.is_empty() {
11093        Vec::new()
11094    } else {
11095        let shell = state.config.shell();
11096        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11097        let cache_dir = state.config.cache_dir();
11098        let (outcomes, _) = with_cache_lease(
11099            state,
11100            cache_dir.as_deref(),
11101            "e2e",
11102            "e2e",
11103            &winner.worktree,
11104            &head,
11105            timeout,
11106            "gate repair e2e",
11107            wait,
11108            |state, budget| async move {
11109                run_e2e_with_retry(
11110                    state,
11111                    &shell,
11112                    &commands,
11113                    &winner.worktree,
11114                    budget,
11115                    "gate repair e2e",
11116                )
11117                .await
11118            },
11119        )
11120        .await?;
11121        outcomes
11122    };
11123    let deferred = verify_inconclusive(&outcomes);
11124    let passed = outcomes.iter().all(CommandOutcome::ok)
11125        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11126    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11127        head,
11128        passed,
11129        outcomes,
11130    });
11131    state.save()?;
11132    if deferred {
11133        Ok(GateFix::Defer)
11134    } else if !passed {
11135        state.event(
11136            "gate",
11137            "verify.e2e failed after gate repair; nothing published",
11138        );
11139        Ok(GateFix::Stop)
11140    } else {
11141        Ok(GateFix::Retry)
11142    }
11143}
11144
11145/// Run the same configured gate for a landing head and a fresh graph.
11146pub(crate) async fn run_landing_gate(
11147    state: &mut RunState,
11148    winner: &Candidate,
11149    wait: &crate::repo_lock::WaitPolicy,
11150) -> Result<Vec<CommandOutcome>> {
11151    let shell = state.config.shell();
11152    let gate_commands = state.config.verify.gate.clone();
11153    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11154    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11155        Vec::new()
11156    } else {
11157        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11158        let cache_dir = state.config.cache_dir();
11159        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11160        let (outcomes, _) = with_cache_lease(
11161            state,
11162            cache_dir.as_deref(),
11163            "gate",
11164            "gate",
11165            &winner.worktree,
11166            &head,
11167            timeout,
11168            "final gate",
11169            wait,
11170            |state, budget| {
11171                let shell = shell.clone();
11172                let gate_commands = gate_commands.clone();
11173                let winner = winner.clone();
11174                async move {
11175                    Runner::run_pre_gate(state, &winner).await;
11176                    let (outcomes, timed_out_pids) = run_commands(
11177                        state,
11178                        "gate",
11179                        "gate",
11180                        0,
11181                        &shell,
11182                        &gate_commands,
11183                        &winner.worktree,
11184                        budget,
11185                    )
11186                    .await;
11187                    (outcomes, false, timed_out_pids)
11188                }
11189            },
11190        )
11191        .await?;
11192        outcomes
11193    };
11194    if outcomes.is_empty() {
11195        // Nothing configured to check — distinct from every other
11196        // silence in this run's event log, since an empty `gate` alone
11197        // no longer says whether the gate ran at all (see
11198        // `RunState::gate_ran`'s own doc).
11199        state.event("gate", "no gate commands configured; not tested");
11200    }
11201    for o in &outcomes {
11202        state.event(
11203            "gate",
11204            format!(
11205                "`{}` -> {}",
11206                o.command,
11207                if o.ok() {
11208                    "pass".to_owned()
11209                } else {
11210                    format!(
11211                        "FAIL ({:?})\n{}",
11212                        o.code,
11213                        tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11214                    )
11215                }
11216            ),
11217        );
11218    }
11219    Ok(outcomes)
11220}
11221
11222/// The pull request title to hand to `land::merge_subject`. A review-only run
11223/// opened by an earlier build titled its pull request with the review prompt;
11224/// that title is dropped (empty, so the fallback applies) rather than landed.
11225/// Any other title, including an operator's rename, passes through untouched,
11226/// and so does every title of a run that implements a task.
11227pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11228    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11229        ""
11230    } else {
11231        pr_title
11232    }
11233}
11234
11235/// What the squash subject falls back to when the pull request title is empty
11236/// or candidate-shaped: for a review-only run the derived title, never the
11237/// review prompt held in `instruction`.
11238pub fn landing_subject_source(state: &RunState) -> String {
11239    if is_review_run(state) {
11240        let winner = state.candidates.first().map_or('A', |c| c.label);
11241        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11242    }
11243    state.instruction.clone()
11244}
11245
11246/// `summary` without its `TITLE:` line, which the pull request title already
11247/// carries.
11248fn summary_without_title(summary: &str) -> String {
11249    let mut lines = summary.trim().lines().peekable();
11250    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11251        lines.next();
11252    }
11253    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11254}
11255
11256/// The pull request title and body for the winning candidate.
11257///
11258/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
11259/// the task's own opening line via [`queue::title_from`] when there is none.
11260/// `state.instruction` can open with blank lines (`task_text` only rejects a
11261/// body that is blank *entirely*), which `title_from` skips.
11262///
11263/// Publishing uses an independent description generated from the final diff
11264/// and structured evidence. Internal summaries and reports remain local.
11265/// Non-publishing previews may display the original task and candidate summary.
11266/// The footer retains the run and candidate provenance.
11267#[cfg(test)]
11268fn pr_message(state: &RunState, winner: char) -> PrMessage {
11269    pr_message_with(state, winner, None)
11270}
11271
11272/// [`pr_message`] with what the branch of a review-only run says about itself.
11273/// `facts` is ignored for a run that implements a task.
11274#[cfg(test)]
11275fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11276    let raw = pr_message_raw(
11277        state,
11278        winner,
11279        facts,
11280        crate::pr_outcome::FinalHead::Unreported,
11281        ChangeNote::NotAsked,
11282    );
11283    let id = crate::scrub::Identity::current();
11284    PrMessage {
11285        title: crate::scrub::scrub(&raw.title, &id),
11286        body: crate::scrub::scrub(&raw.body, &id),
11287    }
11288}
11289
11290/// What the public description may say about the change itself.
11291#[derive(Debug, Clone, Copy)]
11292enum ChangeNote<'a> {
11293    /// A local preview or non-publishing merge mode: no public generation needed.
11294    NotAsked,
11295    /// A summary written from the final diff.
11296    Summary(&'a crate::run::ChangeSummary),
11297    /// A summary was asked for and could not be produced; the text says why.
11298    Unavailable(&'a str),
11299}
11300
11301fn pr_message_raw(
11302    state: &RunState,
11303    winner: char,
11304    facts: Option<&BranchFacts>,
11305    head: crate::pr_outcome::FinalHead<'_>,
11306    change: ChangeNote<'_>,
11307) -> PrMessage {
11308    if !matches!(change, ChangeNote::NotAsked) {
11309        let (title, mut body) = match change {
11310            ChangeNote::Summary(c) => (
11311                c.title.clone(),
11312                format!("## Summary\n\n{}\n\n", c.body.trim()),
11313            ),
11314            ChangeNote::Unavailable(why) => (
11315                neutral_title(state, winner),
11316                format!(
11317                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11318                ),
11319            ),
11320            ChangeNote::NotAsked => unreachable!(),
11321        };
11322        body.push_str(&public_pr_evidence(state, head));
11323        body.push_str(&format!(
11324            "\n---\nmagi:run/{} magi:candidate-{}\n",
11325            state.id,
11326            winner.to_ascii_lowercase()
11327        ));
11328        return PrMessage { title, body };
11329    }
11330    pr_preview(state, winner, facts, head)
11331}
11332
11333/// Append public facts from the run records after every prose generation,
11334/// including posting-gate rewrites. The agent cannot drop this provenance.
11335fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11336    let mut body = crate::pr_outcome::review_rounds(state);
11337    body.push_str(&crate::pr_outcome::final_verification(state, head));
11338    let open = state.last_round_findings();
11339    if !open.is_empty() {
11340        body.push_str("## Open review findings\n\n");
11341        for f in open {
11342            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11343        }
11344        body.push('\n');
11345    }
11346    if let Some(c) = state.change_summary.as_ref()
11347        && head == crate::pr_outcome::FinalHead::At(&c.head)
11348    {
11349        body.push_str(&format!(
11350            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11351            c.base, c.head
11352        ));
11353    }
11354    body
11355}
11356
11357/// Local-only preview of the operator's task and candidate summary.
11358fn pr_preview(
11359    state: &RunState,
11360    winner: char,
11361    facts: Option<&BranchFacts>,
11362    head: crate::pr_outcome::FinalHead<'_>,
11363) -> PrMessage {
11364    let summary = state
11365        .candidates
11366        .iter()
11367        .find(|c| c.label == winner)
11368        .map(|c| c.summary.as_str())
11369        .unwrap_or_default();
11370    // The fallback is the operator's own words and may not be English; GitHub
11371    // text always is, so a non-English task gets a neutral title instead.
11372    let review = is_review_run(state);
11373    let title = if review {
11374        facts
11375            .and_then(|f| {
11376                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11377            })
11378            .or_else(|| review_title(state))
11379            .or_else(|| {
11380                state
11381                    .candidates
11382                    .iter()
11383                    .find(|c| c.label == winner)
11384                    .filter(|c| !c.branch.starts_with("magi/"))
11385                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11386            })
11387            .unwrap_or_else(|| neutral_title(state, winner))
11388    } else {
11389        summary_title(summary).unwrap_or_else(|| {
11390            english_title(
11391                &state.instruction,
11392                PR_TITLE_MAX,
11393                state.config.graph.github_english(),
11394            )
11395            .unwrap_or_else(|| neutral_title(state, winner))
11396        })
11397    };
11398
11399    let mut body = String::new();
11400    let what = summary_without_title(summary);
11401    if !what.is_empty() {
11402        body.push_str("## Summary\n\n");
11403        body.push_str(&what);
11404        body.push_str("\n\n");
11405    }
11406
11407    // Review history uses structured facts across all rounds, never reports.
11408    body.push_str(&crate::pr_outcome::review_rounds(state));
11409    body.push_str(&crate::pr_outcome::final_verification(state, head));
11410    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11411
11412    let open = state.open_findings();
11413    if !open.is_empty() {
11414        body.push_str("## Open review findings\n\n");
11415        for f in &open {
11416            body.push_str(&format!(
11417                "- `{}` [{:?}] {}\n",
11418                f.id, f.severity, "Context retained in local review records."
11419            ));
11420        }
11421        body.push('\n');
11422    }
11423
11424    if let Some(fix) = fix
11425        && !fix.rejected.is_empty()
11426    {
11427        body.push_str("## Declined by the fixer\n\n");
11428        for r in &fix.rejected {
11429            body.push_str(&format!(
11430                "- `{}`: {}\n",
11431                r.id, "Reason retained in local review records."
11432            ));
11433        }
11434        body.push('\n');
11435    }
11436
11437    if review {
11438        // The review prompt is not the task; list what the branch carries.
11439        body.push_str("## Commits under review\n\n");
11440        if let Some(facts) = facts {
11441            let mut left = COMMITS_MAX_CHARS;
11442            for (i, (subject, text)) in facts.commits.iter().enumerate() {
11443                let mut entry = format!("- {}\n", markdown_safe(subject));
11444                for l in markdown_safe(text).lines() {
11445                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
11446                }
11447                if left == 0 {
11448                    body.push_str(&format!(
11449                        "- ... {} more commit(s)\n",
11450                        facts.commits.len() - i
11451                    ));
11452                    break;
11453                }
11454                if entry.len() > left {
11455                    // Even the first commit is cut: one huge body must not
11456                    // push the whole description past GitHub's limit.
11457                    let mut end = left;
11458                    while !entry.is_char_boundary(end) {
11459                        end -= 1;
11460                    }
11461                    entry.truncate(end);
11462                    entry.push_str("\n  ... (truncated)\n");
11463                    left = 0;
11464                } else {
11465                    left -= entry.len();
11466                }
11467                body.push_str(&entry);
11468            }
11469            if !facts.stat.trim().is_empty() {
11470                body.push_str(&format!(
11471                    "\n## Diff stat\n\n```\n{}\n```\n",
11472                    markdown_safe(facts.stat.trim())
11473                ));
11474            }
11475        } else {
11476            match &state.reviewed_commits {
11477                Some(subjects) => {
11478                    for s in subjects {
11479                        body.push_str(&format!("- {}\n", s.trim()));
11480                    }
11481                }
11482                None => {
11483                    // An older run kept only the prompt, with the commit list
11484                    // after its first paragraph.
11485                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11486                    body.push_str(rest.trim());
11487                    body.push('\n');
11488                }
11489            }
11490        }
11491    } else {
11492        let task = state.instruction.trim();
11493        let task = if task.is_empty() {
11494            "(empty task)"
11495        } else {
11496            task
11497        };
11498        body.push_str(&format!(
11499            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11500            task.replace("</details>", "&lt;/details&gt;")
11501        ));
11502    }
11503
11504    body.push_str(&format!(
11505        "\n---\nmagi:run/{} magi:candidate-{}\n",
11506        state.id,
11507        winner.to_ascii_lowercase()
11508    ));
11509
11510    PrMessage { title, body }
11511}
11512
11513/// Source imports have already been prepared from structured inputs. Legacy
11514/// inferred references must never instruct a seat to reconstruct those imports.
11515fn seeded_instruction(state: &RunState) -> String {
11516    let mut inputs = Vec::new();
11517    if let Some(start) = &state.sources.start_from {
11518        inputs.push(format!(
11519            "Start from `{}` at `{}` (including ancestry).",
11520            start.token, start.sha
11521        ));
11522    }
11523    for pick in &state.sources.pick_commits {
11524        inputs.push(format!(
11525            "Pick `{}` at `{}` (single commit).",
11526            pick.token, pick.sha
11527        ));
11528    }
11529    if inputs.is_empty() {
11530        state.instruction.clone()
11531    } else {
11532        format!(
11533            "{}\n\n# Explicit source inputs already prepared\n\n{}",
11534            state.instruction,
11535            inputs.join("\n")
11536        )
11537    }
11538}
11539
11540/// Does the winner have no commits ahead of the base it would land on?
11541/// Any failure to find out reads as "not empty": the merge then behaves as it
11542/// always did rather than refusing on a guess.
11543async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11544    let base = &state.base_branch;
11545    let mut against = base.clone();
11546    if mode == MergeMode::Pr {
11547        // A pull request lands on the remote's base, never the local branch
11548        // of the same name: if that cannot be read, "not empty" is the safe
11549        // answer.
11550        let remote = &state.config.merge.remote;
11551        let tracking = format!("{remote}/{base}");
11552        let fetched = git::fetch(repo, remote, base).await;
11553        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11554            against = tracking;
11555        } else {
11556            return false;
11557        }
11558    }
11559    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11560}
11561
11562/// Why nothing was opened for an empty winner, with what the task's own
11563/// references resolved to.
11564fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11565    let mut detail = format!(
11566        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11567         nothing to open a pull request for"
11568    );
11569    detail.push_str(
11570        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11571    );
11572    detail
11573}
11574
11575/// What the `Pr` merge does once it knows whether the branch already has an
11576/// open pull request.
11577#[derive(Debug, PartialEq, Eq)]
11578enum PrPlan {
11579    Create,
11580    Adopt { url: String, title: String },
11581    Stop(String),
11582}
11583
11584/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
11585/// a failed lookup -> stop with the real reason. Never guesses.
11586fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11587    match found {
11588        Ok(land::OpenPr::None) => PrPlan::Create,
11589        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11590        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11591            "several open pull requests exist for this branch, not picking one: {}",
11592            urls.join(" ")
11593        )),
11594        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11595    }
11596}
11597
11598/// `gh pr create`, returning the PR url.
11599pub(crate) async fn gh_pr_create(
11600    cwd: &Path,
11601    base: &str,
11602    head: &str,
11603    title: &str,
11604    body: &str,
11605) -> Result<String> {
11606    let out = tokio::process::Command::new("gh")
11607        .args([
11608            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11609        ])
11610        .current_dir(cwd)
11611        .quiet()
11612        .stdin(std::process::Stdio::null())
11613        .output()
11614        .await
11615        .context("spawn gh")?;
11616    if out.status.success() {
11617        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11618    } else {
11619        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11620    }
11621}
11622
11623/// Never force deletion of uncommitted work, including unknown legacy trees.
11624async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11625    match std::fs::symlink_metadata(path) {
11626        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11627            // A previous cleanup may have removed the tree before a branch
11628            // deletion or state save failed. Unregister it if still present.
11629            git::worktree_remove_clean(repo, path).await?;
11630            return Ok(());
11631        }
11632        Err(e) => return Err(e.into()),
11633        Ok(_) => {}
11634    }
11635    if !git::is_clean(path).await? {
11636        bail!(
11637            "worktree has uncommitted work; refusing removal: {}",
11638            path.display()
11639        );
11640    }
11641    if !git::worktree_remove_clean(repo, path).await? {
11642        bail!("worktree removal refused: {}", path.display());
11643    }
11644    Ok(())
11645}
11646
11647/// Tear a run's worktrees and branches down.
11648///
11649/// `home` is where the updated `run.json` is saved (via
11650/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
11651/// a housekeeping pass already has its own honest `home` handed to it, and
11652/// falling through to the global here would write back through whichever
11653/// directory some other process or test pinned into that `OnceLock` first,
11654/// not the one the caller actually resolved its `runs` and `state` from.
11655pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11656    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11657        bail!("refusing fold: {reason}");
11658    }
11659    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11660        bail!("candidate preservation incomplete; refusing cleanup");
11661    }
11662    // Serialize ordinary folding with the same starts/resumes as target-only
11663    // eviction. Otherwise a second janitor could remove the whole worktree
11664    // around a live output borrower even though target eviction was fenced.
11665    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11666    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11667    let _driver = crate::driver::maintenance(home, &state.id)?;
11668    let mut output_guards = Vec::new();
11669    for entry in std::fs::read_dir(state.worktree_root())
11670        .into_iter()
11671        .flatten()
11672        .flatten()
11673    {
11674        if entry.path().join(".git").is_file() {
11675            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11676        }
11677    }
11678    let repo = state.repo.clone();
11679    // CLI, Web and automatic cleanup all enter here, independently of
11680    // Runner::fold_losers. Protect deletion and shared config release too.
11681    let wait = repository_wait(state, None);
11682    let _coordination = match coordinate_operation(state, &repo, &wait).await {
11683        Ok(lease) => lease,
11684        Err(e) => {
11685            state.save_under(home)?;
11686            return Err(e);
11687        }
11688    };
11689    // Confirmed merges dispose of their owned sources after durable evidence
11690    // capture. Preflight every source before removing anything.
11691    let _source_outputs = if state.status == RunStatus::Merged
11692        && state.merge.as_ref().is_some_and(|merge| merge.ok)
11693    {
11694        verify_merged_fold_sources(state, drop_winner, home).await?
11695    } else {
11696        Vec::new()
11697    };
11698    let root = state.worktree_root();
11699    let winner = state.tally.as_ref().map(|t| t.winner);
11700    let mut removed = Vec::new();
11701
11702    for i in 0..state.candidates.len() {
11703        let c = state.candidates[i].clone();
11704        let is_winner = Some(c.label) == winner;
11705        if is_winner && !drop_winner {
11706            continue;
11707        }
11708        let existed = c.worktree.exists();
11709        // A vanished directory can still have a Git registration that keeps
11710        // its branch checked out. Remove that registration before the branch.
11711        remove_preserved_worktree(&repo, &c.worktree).await?;
11712        if existed {
11713            removed.push(c.worktree.to_string_lossy().into_owned());
11714        }
11715        // A branch handed to a later run (and its pull request) is not this
11716        // run's to delete.
11717        let handed_over = state.released_branches.contains(&c.branch);
11718        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11719            if !git::branch_delete(&repo, &c.branch).await? {
11720                bail!("candidate branch removal refused");
11721            }
11722            removed.push(c.branch.clone());
11723        }
11724        state.candidates[i].folded = true;
11725    }
11726
11727    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11728        let path = name.path();
11729        let keep = !drop_winner
11730            && winner.is_some_and(|w| {
11731                path.file_name()
11732                    .is_some_and(|n| n == format!("cand-{w}").as_str())
11733            });
11734        if keep {
11735            continue;
11736        }
11737        remove_preserved_worktree(&repo, &path).await?;
11738        removed.push(path.to_string_lossy().into_owned());
11739    }
11740
11741    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
11742    // judge worktrees, so once the loop above has cleared all of them out,
11743    // the parent is a bare directory nobody else was ever going to remove -
11744    // git only ever managed what was inside it. Left alone, one of these
11745    // accumulates per fully-folded run; the operator's own machine had 74.
11746    // `remove_if_empty` re-checks rather than assuming: a run whose winner
11747    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
11748    // so does anything a run never claimed that happens to share the bay.
11749    remove_if_empty(&root);
11750
11751    if state.enabled_worktree_config && drop_winner {
11752        // A release, not a raw disable: some sibling run in this repository
11753        // may still hold its own reference (see `git::acquire_worktree_config`),
11754        // and only the last release actually turns the setting back off.
11755        git::release_worktree_config(&repo).await.ok();
11756        state.enabled_worktree_config = false;
11757    }
11758    state.save_under(home)?;
11759    Ok(removed)
11760}
11761
11762async fn verify_merged_fold_sources(
11763    state: &RunState,
11764    drop_winner: bool,
11765    home: &Path,
11766) -> Result<Vec<std::fs::File>> {
11767    let winner = state.tally.as_ref().map(|t| t.winner);
11768    let mut heads = BTreeSet::new();
11769    for candidate in &state.candidates {
11770        if Some(candidate.label) == winner && !drop_winner {
11771            continue;
11772        }
11773        anyhow::ensure!(
11774            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11775            "candidate recovery source is outside the inventoried run root: {}",
11776            candidate.worktree.display()
11777        );
11778        match std::fs::symlink_metadata(&candidate.worktree) {
11779            Ok(_) => {}
11780            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11781            Err(error) => return Err(error.into()),
11782        }
11783        if !state.released_branches.contains(&candidate.branch)
11784            && git::branch_exists(&state.repo, &candidate.branch).await?
11785        {
11786            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11787        }
11788    }
11789    let mut cargo_guards = Vec::new();
11790    let entries = match std::fs::read_dir(state.worktree_root()) {
11791        Ok(entries) => Some(entries),
11792        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11793        Err(error) => return Err(error.into()),
11794    };
11795    for entry in entries.into_iter().flatten() {
11796        let path = entry?.path();
11797        let metadata = std::fs::symlink_metadata(&path)?;
11798        anyhow::ensure!(
11799            metadata.is_dir() && !metadata.file_type().is_symlink(),
11800            "non-directory or symlink recovery source retained: {}",
11801            path.display()
11802        );
11803        if !drop_winner
11804            && winner.is_some_and(|w| {
11805                path.file_name()
11806                    .is_some_and(|name| name == format!("cand-{w}").as_str())
11807            })
11808        {
11809            continue;
11810        }
11811        anyhow::ensure!(
11812            path.join(".git").is_file(),
11813            "unregistered recovery source retained: {}",
11814            path.display()
11815        );
11816        anyhow::ensure!(
11817            git::git(
11818                &path,
11819                &[
11820                    "status",
11821                    "--porcelain",
11822                    "--untracked-files=all",
11823                    "--ignored",
11824                    "--",
11825                    ".",
11826                    ":(exclude)target"
11827                ]
11828            )
11829            .await?
11830            .is_empty(),
11831            "source changes or ignored recovery files retained: {}",
11832            path.display()
11833        );
11834        match std::fs::symlink_metadata(path.join("target")) {
11835            Ok(_) => {
11836                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11837            }
11838            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11839            Err(error) => return Err(error.into()),
11840        }
11841        for checkpoint in [
11842            "rebase-merge",
11843            "rebase-apply",
11844            "sequencer",
11845            "CHERRY_PICK_HEAD",
11846            "MERGE_HEAD",
11847            "REBASE_HEAD",
11848            "REVERT_HEAD",
11849            "BISECT_LOG",
11850            "AUTO_MERGE",
11851            "MERGE_MSG",
11852        ] {
11853            let location = git::git(
11854                &path,
11855                &[
11856                    "rev-parse",
11857                    "--path-format=absolute",
11858                    "--git-path",
11859                    checkpoint,
11860                ],
11861            )
11862            .await?;
11863            anyhow::ensure!(
11864                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11865                if e.kind() == std::io::ErrorKind::NotFound),
11866                "Git recovery checkpoint retained: {checkpoint}"
11867            );
11868        }
11869        heads.insert(git::rev_parse(&path, "HEAD").await?);
11870    }
11871    crate::fold_evidence::preserve(state, home, &heads).await?;
11872
11873    Ok(cargo_guards)
11874}
11875
11876/// Remove `dir` if it exists and has nothing in it.
11877///
11878/// Best-effort and silent by design: a directory that is not empty (a run
11879/// whose winner is still parked there, a stray file some other process left)
11880/// is exactly the case this must refuse, and a directory that is already gone
11881/// is not a failure worth reporting either. `std::fs::remove_dir` itself
11882/// already refuses a non-empty directory, so the emptiness check below is
11883/// belt, not suspenders - it is what keeps this from ever attempting the
11884/// removal in the case that matters, rather than trusting `remove_dir`'s
11885/// error path to have no side effects if it ever changed.
11886fn remove_if_empty(dir: &Path) {
11887    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11888        std::fs::remove_dir(dir).ok();
11889    }
11890}
11891
11892/// Severity of the worst open finding in the last review round, for reporting.
11893pub fn worst_open(state: &RunState) -> Option<Severity> {
11894    state
11895        .reviews
11896        .last()?
11897        .reviews
11898        .iter()
11899        .flat_map(|r| r.findings.iter())
11900        .map(|f| f.severity)
11901        .max()
11902}
11903
11904/// `[worktree] setup` hides its products per worktree, which needs
11905/// `extensions.worktreeConfig`. Taken once per run and released with the run's
11906/// worktrees, like the hook's; every place that runs setup calls this first.
11907async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11908    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11909        git::acquire_worktree_config(repo).await?;
11910        state.enabled_worktree_config = true;
11911    }
11912    Ok(())
11913}
11914
11915#[cfg(test)]
11916mod tests {
11917    #[tokio::test]
11918    async fn launcher_base_fetch_requires_remote_success() {
11919        let tmp = tempfile::tempdir().unwrap();
11920        let repo = tmp.path().join("repo");
11921        let remote = tmp.path().join("remote");
11922        std::fs::create_dir(&repo).unwrap();
11923        std::fs::create_dir(&remote).unwrap();
11924        git::git(&remote, &["init", "--bare", "-b", "main"])
11925            .await
11926            .unwrap();
11927        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11928        git::git(
11929            &repo,
11930            &[
11931                "-c",
11932                "user.name=Fixture",
11933                "-c",
11934                "user.email=fixture@example.invalid",
11935                "commit",
11936                "--allow-empty",
11937                "-m",
11938                "fixture",
11939            ],
11940        )
11941        .await
11942        .unwrap();
11943        git::git(
11944            &repo,
11945            &["remote", "add", "origin", remote.to_str().unwrap()],
11946        )
11947        .await
11948        .unwrap();
11949        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11950        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11951        assert_eq!(
11952            resolve_base(&repo, "main", "origin", None)
11953                .await
11954                .unwrap()
11955                .commit,
11956            head
11957        );
11958        #[cfg(unix)]
11959        {
11960            let policy: crate::config::Coordination = toml::from_str(
11961                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11962            ).unwrap();
11963            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11964            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11965            let held = crate::repo_lock::SeatLease::operation_with(
11966                &repo,
11967                Some(&policy),
11968                &crate::repo_lock::WaitPolicy::default(),
11969            )
11970            .await
11971            .unwrap();
11972            assert!(held.is_some());
11973            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11974            tokio::pin!(fetch);
11975            assert!(
11976                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11977                    .await
11978                    .is_err()
11979            );
11980            assert!(
11981                !repo.join(".git/FETCH_HEAD").exists(),
11982                "no fetch before admission"
11983            );
11984            drop(held);
11985            let fetched = fetch.await.unwrap();
11986            assert_eq!(fetched.commit, head);
11987            assert_eq!(fetched.remote, "origin");
11988            assert_eq!(fetched.branch, "main");
11989        }
11990        // A surviving tracking ref is not successful fetch evidence.
11991        git::git(
11992            &repo,
11993            &[
11994                "remote",
11995                "set-url",
11996                "origin",
11997                tmp.path().join("missing").to_str().unwrap(),
11998            ],
11999        )
12000        .await
12001        .unwrap();
12002        let error = resolve_base(&repo, "main", "origin", None)
12003            .await
12004            .unwrap_err()
12005            .to_string();
12006        assert!(error.contains("cannot read origin/main"));
12007        assert!(error.contains("refusing to branch off"));
12008    }
12009
12010    #[test]
12011    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12012        let leaked = vec!["chore(deps): update a crate".to_owned()];
12013        let own = "fix(daemon): apply a chosen action";
12014        assert!(should_retitle("", own, &leaked));
12015        assert!(should_retitle(
12016            &format!("{REVIEW_PROMPT_OPENING} `x`"),
12017            own,
12018            &leaked
12019        ));
12020        assert!(should_retitle(
12021            "chore: land candidate A of run 1",
12022            own,
12023            &leaked
12024        ));
12025        assert!(should_retitle(
12026            "magi: candidate A (uncommitted work)",
12027            own,
12028            &leaked
12029        ));
12030        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12031        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12032        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12033    }
12034
12035    #[test]
12036    fn pr_merge_plan_creates_adopts_or_stops() {
12037        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12038        assert_eq!(
12039            pr_merge_plan(Ok(land::OpenPr::One {
12040                url: "u".into(),
12041                title: "t".into()
12042            })),
12043            PrPlan::Adopt {
12044                url: "u".into(),
12045                title: "t".into()
12046            }
12047        );
12048        let PrPlan::Stop(many) =
12049            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12050        else {
12051            panic!("many must stop");
12052        };
12053        assert!(many.contains('a') && many.contains('b'));
12054        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12055            panic!("a failed lookup must stop");
12056        };
12057        assert!(err.contains("bad token"));
12058    }
12059
12060    use super::*;
12061    use crate::run::GateStatus;
12062    use std::collections::BTreeMap;
12063    use std::time::Duration;
12064
12065    fn conductor() -> AgentSpec {
12066        AgentSpec {
12067            id: "conductor".to_owned(),
12068            kind: crate::config::AgentKind::Command,
12069            model: None,
12070            command: vec!["true".to_owned()],
12071            extra_args: Vec::new(),
12072            env: BTreeMap::new(),
12073            prompt_delivery: None,
12074            max_turn_secs: None,
12075        }
12076    }
12077
12078    fn spec(id: &str) -> AgentSpec {
12079        AgentSpec {
12080            id: id.to_owned(),
12081            kind: crate::config::AgentKind::Command,
12082            model: None,
12083            command: vec!["true".to_owned()],
12084            extra_args: Vec::new(),
12085            env: BTreeMap::new(),
12086            prompt_delivery: None,
12087            max_turn_secs: None,
12088        }
12089    }
12090
12091    fn ids(xs: &[&str]) -> BTreeSet<String> {
12092        xs.iter().map(|s| (*s).to_owned()).collect()
12093    }
12094
12095    #[test]
12096    fn pick_successor_skips_an_agent_another_seat_holds() {
12097        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
12098        let roster = [spec("a"), spec("b"), spec("c")];
12099        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12100        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12101    }
12102
12103    #[test]
12104    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12105        // The other seat started on c but was handed to d; c is free again.
12106        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12107        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12108        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12109        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12110        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12111    }
12112
12113    #[test]
12114    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12115        let roster = [spec("a"), spec("b")];
12116        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12117        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12118        let carried = ids(&["b"]);
12119        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12120        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12121    }
12122
12123    #[test]
12124    fn pick_successor_returns_none_without_any_untried_successor() {
12125        let roster = [spec("a"), spec("b")];
12126        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12127        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12128        let carried = ids(&["a"]);
12129        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12130    }
12131
12132    #[test]
12133    fn pick_successor_rescue_avoids_another_seats_occupant() {
12134        // b is a carried failure and free; a is held by the other seat.
12135        let roster = [spec("a"), spec("b"), spec("c")];
12136        let carried = ids(&["b", "c"]);
12137        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12138        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12139    }
12140
12141    #[test]
12142    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12143        let roster = [spec("a"), spec("b"), spec("c")];
12144        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12145        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12146    }
12147
12148    #[test]
12149    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12150        let roster = [spec("a"), spec("b"), spec("c")];
12151        let failed = ids(&["a", "b", "c"]);
12152        // Rescue looks at the whole roster, once per id, then runs out.
12153        let mut tried = ids(&["b"]);
12154        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12155        assert_eq!(first.id, "a");
12156        tried.insert(first.id.clone());
12157        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12158        assert_eq!(second.id, "c");
12159        tried.insert(second.id.clone());
12160        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12161    }
12162
12163    #[test]
12164    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12165        let roster = [spec("a"), spec("b")];
12166        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12167        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12168    }
12169
12170    #[test]
12171    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12172        let roster = [spec("a"), spec("b"), spec("c")];
12173        let h = SeatHistory {
12174            failed: ids(&["a"]),
12175            last_ok: Some("b".to_owned()),
12176            last_fail: None,
12177        };
12178        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12179        // A last answerer that left the roster, or later failed, is ignored.
12180        let gone = SeatHistory {
12181            last_ok: Some("zzz".to_owned()),
12182            ..h.clone()
12183        };
12184        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12185        let failed = SeatHistory {
12186            failed: ids(&["a", "b"]),
12187            last_ok: Some("b".to_owned()),
12188            last_fail: None,
12189        };
12190        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12191    }
12192
12193    #[test]
12194    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12195        let first = SeatState::new("review-1", "alpha", 7);
12196        let next = handover_seat("review-1", "gamma", 7);
12197        assert_ne!(first.claude_session, next.claude_session);
12198    }
12199
12200    #[test]
12201    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12202        let mut state = state_with_summary("x", "y");
12203        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12204        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12205        assert_ne!(a.claude_session, b.claude_session);
12206    }
12207
12208    #[test]
12209    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12210        let roster = [spec("a"), spec("b")];
12211        let h = SeatHistory {
12212            failed: ids(&["a", "b"]),
12213            ..SeatHistory::default()
12214        };
12215        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12216        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12217        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12218    }
12219
12220    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12221    // fallback loop depends on to terminate: it must walk forward from the
12222    // seat's own position, never restart at the front of the roster, and it
12223    // must never hand back an id already tried, however many times that id
12224    // happens to appear.
12225
12226    #[test]
12227    fn failure_signature_ignores_numbers_and_paths() {
12228        assert_eq!(
12229            failure_signature("exited with Some(2) and no usable output"),
12230            failure_signature("exited with Some(137) and no usable output")
12231        );
12232        assert_eq!(
12233            failure_signature("process exited with no usable output (exit code 2)"),
12234            failure_signature("process exited with no usable output (exit code 137)")
12235        );
12236        for (old, new) in [
12237            (
12238                "exited with Some(2) and no usable output",
12239                "process exited with no usable output (exit code 2)",
12240            ),
12241            (
12242                "exited with None and no usable output",
12243                "process exited with no usable output (exit code unavailable)",
12244            ),
12245        ] {
12246            let previous = FailClass::Other(failure_signature(old));
12247            let current = FailClass::Other(failure_signature(new));
12248            assert_eq!(previous, current, "resumption retains the failure class");
12249            assert!(!should_hand_over(Some(&previous), &current));
12250        }
12251        assert_eq!(
12252            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12253            failure_signature("cannot open /var/x.txt: denied")
12254        );
12255        assert_ne!(failure_signature("boom"), failure_signature("bang"));
12256    }
12257
12258    #[test]
12259    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12260        let other = FailClass::Other("x".into());
12261        assert!(should_hand_over(None, &FailClass::Quota));
12262        assert!(should_hand_over(Some(&other), &FailClass::Quota));
12263        assert!(should_hand_over(
12264            Some(&FailClass::Timeout),
12265            &FailClass::Timeout
12266        ));
12267        assert!(should_hand_over(None, &other));
12268        assert!(!should_hand_over(Some(&other), &other));
12269        assert!(should_hand_over(
12270            Some(&other),
12271            &FailClass::Other("y".into())
12272        ));
12273        // A quota or timeout in between ends the run of identical failures.
12274        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12275        assert!(should_hand_over(Some(&FailClass::Quota), &other));
12276    }
12277
12278    #[test]
12279    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12280        let a = SeatState::new("judge-1", "alpha", 7);
12281        let b = handover_seat("judge-1", "beta", 7);
12282        assert_ne!(a.claude_session, b.claude_session);
12283        assert_eq!(b.turns, 0);
12284    }
12285
12286    #[test]
12287    fn a_timeout_is_classified_apart_from_other_failures() {
12288        assert_eq!(
12289            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12290            Some(FailClass::Timeout)
12291        );
12292        assert!(matches!(
12293            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12294            Some(FailClass::Other(_))
12295        ));
12296    }
12297
12298    #[test]
12299    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12300        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12301        let tried = BTreeSet::from(["beta".to_owned()]);
12302        // beta sits at index 1; the next candidate is gamma, never alpha —
12303        // which is very likely a different candidate slot's own agent.
12304        let next = next_untried_in_roster(&roster, 1, &tried);
12305        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12306    }
12307
12308    #[test]
12309    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12310        let roster = vec![spec("alpha"), spec("beta")];
12311        let tried = BTreeSet::from(["beta".to_owned()]);
12312        // beta is the roster's last entry: nothing follows it, and alpha —
12313        // earlier in the roster, almost certainly a different candidate
12314        // slot's own agent — must not be reached by wrapping back to it.
12315        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12316    }
12317
12318    #[test]
12319    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12320        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12321        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12322        // beta (index 1) and gamma (index 2, the only entry after it) have
12323        // both been tried; alpha (index 0) never has, but it comes before
12324        // beta's own position, so there is nothing further for this seat.
12325        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12326    }
12327
12328    #[test]
12329    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12330        let roster = vec![spec("a"), spec("a"), spec("b")];
12331        let tried = BTreeSet::from(["a".to_owned()]);
12332        let next = next_untried_in_roster(&roster, 0, &tried);
12333        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12334    }
12335
12336    #[test]
12337    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12338        let roster = vec![spec("a"), spec("b")];
12339        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12340        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12341    }
12342
12343    #[test]
12344    fn remove_if_empty_only_ever_takes_a_bare_directory() {
12345        let dir = tempfile::tempdir().unwrap();
12346        let bay = dir.path().join("ffff");
12347
12348        // Not there yet: nothing to do, nothing to panic on.
12349        remove_if_empty(&bay);
12350        assert!(!bay.exists());
12351
12352        // Something still inside - the winner's worktree, or a stray file -
12353        // keeps the directory standing.
12354        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12355        remove_if_empty(&bay);
12356        assert!(bay.exists(), "non-empty directory must survive");
12357
12358        // Once the last entry is gone, so is the directory itself.
12359        std::fs::remove_dir(bay.join("cand-A")).unwrap();
12360        remove_if_empty(&bay);
12361        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12362    }
12363
12364    // `round_is_clean` is the exact decision this task fixed: a round with a
12365    // seat that never answered must not read the same as a round every seat
12366    // actually reviewed. These are deterministic and process-free by design —
12367    // the equivalent end-to-end check (a real reviewer timing out under a
12368    // live graph run) is a genuine race against wall-clock contention, and a
12369    // spawn slow enough to blow even a generous budget under a loaded test
12370    // run must not turn this specific regression check flaky.
12371
12372    #[test]
12373    fn a_full_panel_that_found_nothing_is_clean() {
12374        assert!(round_is_clean(
12375            0,
12376            true,
12377            2,
12378            2,
12379            0,
12380            IncompleteReviewPolicy::Block
12381        ));
12382    }
12383
12384    #[test]
12385    fn a_missing_seat_is_never_clean_under_the_default_policy() {
12386        assert!(!round_is_clean(
12387            0,
12388            true,
12389            1,
12390            2,
12391            0,
12392            IncompleteReviewPolicy::Block
12393        ));
12394    }
12395
12396    #[test]
12397    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12398        assert!(!round_is_clean(
12399            1,
12400            true,
12401            1,
12402            2,
12403            0,
12404            IncompleteReviewPolicy::Warn
12405        ));
12406    }
12407
12408    #[test]
12409    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12410        assert!(round_is_clean(
12411            0,
12412            true,
12413            1,
12414            2,
12415            0,
12416            IncompleteReviewPolicy::Warn
12417        ));
12418    }
12419
12420    #[test]
12421    fn a_full_panel_with_an_open_finding_is_not_clean() {
12422        assert!(!round_is_clean(
12423            1,
12424            true,
12425            2,
12426            2,
12427            0,
12428            IncompleteReviewPolicy::Block
12429        ));
12430    }
12431
12432    #[test]
12433    fn a_full_panel_with_a_red_e2e_is_not_clean() {
12434        assert!(!round_is_clean(
12435            0,
12436            false,
12437            2,
12438            2,
12439            0,
12440            IncompleteReviewPolicy::Block
12441        ));
12442    }
12443
12444    // The stall this task closes: under the default `block` policy, a seat
12445    // missing only because it was rate limited must not force a wait for a
12446    // session limit that will not lift by the next round. `round_is_clean`
12447    // is where that quorum carve-out lives; the review loop around it never
12448    // changes what a reviewer's vote or a finding's severity means.
12449
12450    #[test]
12451    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12452        // 1 of 2 answered, and the one missing was quota'd — the exact
12453        // "review-2 rate limited (quota)" shape from the field report.
12454        assert!(round_is_clean(
12455            0,
12456            true,
12457            1,
12458            2,
12459            1,
12460            IncompleteReviewPolicy::Block
12461        ));
12462    }
12463
12464    #[test]
12465    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12466        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
12467        // not a quota loss (`quota_missing` stays 0) — worth another try.
12468        assert!(!round_is_clean(
12469            0,
12470            true,
12471            1,
12472            2,
12473            0,
12474            IncompleteReviewPolicy::Block
12475        ));
12476    }
12477
12478    #[test]
12479    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12480        assert!(!round_is_clean(
12481            1,
12482            true,
12483            1,
12484            2,
12485            1,
12486            IncompleteReviewPolicy::Block
12487        ));
12488        assert!(!round_is_clean(
12489            0,
12490            false,
12491            1,
12492            2,
12493            1,
12494            IncompleteReviewPolicy::Block
12495        ));
12496    }
12497
12498    #[test]
12499    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12500        // Every seat quota'd, nobody answered: there is no panel to decide
12501        // on, so this must fall through to the existing block-and-retry
12502        // fallback rather than call an unreviewed patch clean.
12503        assert!(!round_is_clean(
12504            0,
12505            true,
12506            0,
12507            2,
12508            2,
12509            IncompleteReviewPolicy::Block
12510        ));
12511    }
12512
12513    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12514        CommandOutcome {
12515            command: "test".to_owned(),
12516            code,
12517            output_tail: String::new(),
12518            duration_ms: 0,
12519            resource_blocked,
12520        }
12521    }
12522
12523    #[test]
12524    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12525        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12526        assert!(
12527            !verify_inconclusive(&[outcome(Some(1), false)]),
12528            "an ordinary failure is still evidence about the patch"
12529        );
12530        assert!(verify_inconclusive(&[outcome(None, true)]));
12531        assert!(
12532            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12533            "one inconclusive outcome taints the whole batch"
12534        );
12535        assert!(!verify_inconclusive(&[]));
12536    }
12537
12538    #[tokio::test]
12539    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12540        // Alive for the first two checks, then dead - confirms the loop
12541        // actually re-polls rather than deciding once and sleeping out the
12542        // ceiling regardless.
12543        let calls = std::sync::atomic::AtomicUsize::new(0);
12544        let started = Instant::now();
12545        wait_for_pids_with(
12546            &[123],
12547            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12548            Duration::from_millis(5),
12549            Duration::from_secs(5),
12550        )
12551        .await;
12552        assert!(
12553            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12554            "must keep checking rather than deciding on the first answer"
12555        );
12556        assert!(
12557            started.elapsed() < Duration::from_secs(1),
12558            "must return the moment it is confirmed dead, not wait out the ceiling"
12559        );
12560    }
12561
12562    #[tokio::test]
12563    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12564        let started = Instant::now();
12565        wait_for_pids_with(
12566            &[123],
12567            |_| true, // never reports dead
12568            Duration::from_millis(5),
12569            Duration::from_millis(30),
12570        )
12571        .await;
12572        let elapsed = started.elapsed();
12573        assert!(
12574            elapsed >= Duration::from_millis(30),
12575            "must not give up before its own ceiling: {elapsed:?}"
12576        );
12577        assert!(
12578            elapsed < Duration::from_secs(1),
12579            "must not wait past its own ceiling either: {elapsed:?}"
12580        );
12581    }
12582
12583    #[tokio::test]
12584    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12585        let started = Instant::now();
12586        wait_for_pids_with(
12587            &[],
12588            |_| true,
12589            Duration::from_secs(5),
12590            Duration::from_secs(5),
12591        )
12592        .await;
12593        assert!(
12594            started.elapsed() < Duration::from_millis(200),
12595            "an empty pid list has nothing to confirm"
12596        );
12597    }
12598
12599    // `review_conclusion` is the exact decision the review hand-off task
12600    // fixed: a round budget spent (or a tree that stopped moving) must not
12601    // collapse into `Blocked` regardless of what verification actually
12602    // said. Deterministic and process-free for the same reason the
12603    // `round_is_clean` family above is.
12604    fn review_round(
12605        clean: bool,
12606        blocking: usize,
12607        answered: usize,
12608        expected: usize,
12609        progressed: bool,
12610        e2e_ok: bool,
12611    ) -> ReviewRound {
12612        ReviewRound {
12613            round: 1,
12614            head: "h".to_owned(),
12615            verified_head: None,
12616            verified_at: None,
12617            reviews: Vec::new(),
12618            e2e: vec![CommandOutcome {
12619                command: "test".to_owned(),
12620                code: Some(if e2e_ok { 0 } else { 1 }),
12621                output_tail: String::new(),
12622                duration_ms: 0,
12623                resource_blocked: false,
12624            }],
12625            verify_retried: false,
12626            e2e_deferred: false,
12627            e2e_defer_reason: None,
12628            fix: None,
12629            blocking,
12630            answered,
12631            expected,
12632            clean,
12633            progressed,
12634            vote_split: false,
12635            reconsideration: Vec::new(),
12636            verdict: None,
12637        }
12638    }
12639
12640    #[test]
12641    fn review_conclusion_is_none_when_nothing_has_run() {
12642        assert_eq!(review_conclusion(&[], 3), None);
12643    }
12644
12645    #[test]
12646    fn review_conclusion_is_none_while_rounds_remain() {
12647        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12648        assert_eq!(review_conclusion(&rounds, 3), None);
12649    }
12650
12651    #[test]
12652    fn review_conclusion_is_gating_once_a_round_is_clean() {
12653        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12654        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12655    }
12656
12657    #[test]
12658    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12659        let rounds = vec![
12660            review_round(false, 1, 2, 2, true, true),
12661            review_round(false, 1, 2, 2, true, true),
12662        ];
12663        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12664    }
12665
12666    #[test]
12667    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12668        let rounds = vec![
12669            review_round(false, 1, 2, 2, true, true),
12670            review_round(false, 1, 2, 2, true, false),
12671        ];
12672        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12673    }
12674
12675    #[test]
12676    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12677        // Magi never got a command to run against this round's own head — a
12678        // resource-blocked attempt, not a red one — so this must never
12679        // settle on `Blocked` the way a genuine e2e failure would. `None`
12680        // here is what tells `Runner::review_loop` to retry the check
12681        // itself rather than trust this cheap recomputation with a verdict
12682        // it cannot actually produce.
12683        let mut blocked = review_round(false, 1, 2, 2, true, false);
12684        blocked.e2e[0].resource_blocked = true;
12685        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12686        assert_eq!(review_conclusion(&rounds, 2), None);
12687    }
12688
12689    #[test]
12690    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12691        // Missing input, not a verified tree — never a hand-off candidate.
12692        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12693        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12694    }
12695
12696    #[test]
12697    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12698        let rounds = vec![
12699            review_round(false, 1, 2, 2, false, true),
12700            review_round(false, 1, 2, 2, false, true),
12701        ];
12702        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12703    }
12704
12705    fn secs(n: u64) -> Duration {
12706        Duration::from_secs(n)
12707    }
12708
12709    /// A throwaway repo with one commit on `main`, for tests that need `merge`
12710    /// to make real (and, if it runs at all, real*ly fail*) git calls.
12711    #[tokio::test]
12712    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12713        ask_test_home();
12714        let tmp = tempfile::tempdir().unwrap();
12715        let repo = tmp.path().join("repo");
12716        std::fs::create_dir_all(&repo).unwrap();
12717        init_repo(&repo);
12718        let g = |args: &[&str]| {
12719            let out = std::process::Command::new("git")
12720                .args(args)
12721                .current_dir(&repo)
12722                .quiet()
12723                .output()
12724                .unwrap();
12725            assert!(
12726                out.status.success(),
12727                "{}",
12728                String::from_utf8_lossy(&out.stderr)
12729            );
12730        };
12731        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12732        g(&["checkout", "-b", "historical"]);
12733        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12734        g(&["add", "."]);
12735        g(&["commit", "-m", "old alternative"]);
12736        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12737        g(&["checkout", "-b", "magi/task-selected", &base]);
12738        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12739        g(&["add", "."]);
12740        g(&["commit", "-m", "measurement infrastructure"]);
12741        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12742        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12743        g(&["add", "."]);
12744        g(&["commit", "-m", "dependent implementation"]);
12745        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12746        g(&["checkout", "main"]);
12747        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12748        g(&["add", "."]);
12749        g(&["commit", "-m", "unrelated base change"]);
12750        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12751        let origin = tmp.path().join("origin.git");
12752        g(&[
12753            "clone",
12754            "--bare",
12755            repo.to_str().unwrap(),
12756            origin.to_str().unwrap(),
12757        ]);
12758        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12759        g(&["fetch", "origin"]);
12760        let selection = crate::pr_recovery::Selection {
12761            url: "https://example.test/project/pull/1".into(),
12762            branch: "magi/task-selected".into(),
12763            head: head.clone(),
12764            source_head: None,
12765            source_run: None,
12766            source_proof: None,
12767            disposition: Some(crate::pr_recovery::Disposition::Review),
12768            instruction: "preserve measured behavior".into(),
12769            approved_at: jiff::Timestamp::now(),
12770        };
12771        let mut template = runner_at(RunStatus::Reviewing);
12772        template.state.repo = repo.clone();
12773        template.state.base_commit = current.clone();
12774        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12775        template.state.origin = Some(crate::run::Origin::queue("selected"));
12776        template.state.pr_head = Some(selection.branch.clone());
12777        template.state.pr_recovery = Some(selection.clone());
12778        let mut runner =
12779            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12780                .await
12781                .unwrap();
12782        let wt = runner.state.winner().unwrap().worktree.clone();
12783        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12784        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12785        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12786        assert!(runner.state.seeds.is_empty());
12787        assert_eq!(runner.state.instruction, "preserve measured behavior");
12788        runner.sync_to_base().await.unwrap();
12789        assert_eq!(
12790            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12791            "prerequisite"
12792        );
12793        assert_eq!(
12794            std::fs::read_to_string(wt.join("implementation")).unwrap(),
12795            "measured implementation"
12796        );
12797        assert_eq!(
12798            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12799            "current main change"
12800        );
12801        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12802        assert_eq!(
12803            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12804            head
12805        );
12806
12807        // A failed selected-source setup stays inspectable, never resumable as a new competition.
12808        let mut partial = runner_at(RunStatus::Reviewing);
12809        partial.state.repo = repo.clone();
12810        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12811        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12812            run: Some("exit 1".into()),
12813            ..Default::default()
12814        }];
12815        let selection = runner.state.pr_recovery.clone().unwrap();
12816        partial.state.pr_recovery = Some(selection.clone());
12817        partial.state.pr_head = Some(selection.branch.clone());
12818        partial.state.origin = Some(crate::run::Origin::queue("selected"));
12819        let id = partial.state.id.clone();
12820        let error = match Runner::open_selected(
12821            &repo,
12822            partial.state,
12823            partial.roles,
12824            1,
12825            runner.state.base_commit.clone(),
12826            selection,
12827        )
12828        .await
12829        {
12830            Ok(_) => panic!("setup must fail"),
12831            Err(e) => e,
12832        };
12833        assert_eq!(
12834            error
12835                .downcast_ref::<crate::pr_recovery::StartupFailed>()
12836                .unwrap()
12837                .run,
12838            id
12839        );
12840        let failed = RunState::load(&id).unwrap();
12841        assert_eq!(failed.status, RunStatus::Blocked);
12842        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12843        assert!(Runner::resume(&id).is_err());
12844        runner.state.status = RunStatus::Ready;
12845        runner.state.driver_exited = true;
12846        runner.state.save().unwrap();
12847        let mut retry = runner_at(RunStatus::Reviewing);
12848        retry.state.repo = repo.clone();
12849        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12850        let mut reselected = runner.state.pr_recovery.clone().unwrap();
12851        reselected.approved_at = jiff::Timestamp::now();
12852        retry.state.origin = Some(crate::run::Origin::queue("selected"));
12853        retry.state.pr_head = Some(reselected.branch.clone());
12854        retry.state.pr_recovery = Some(reselected.clone());
12855        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12856            .await
12857            .unwrap();
12858        let restarted = Runner::open_selected(
12859            &repo,
12860            retry.state,
12861            retry.roles,
12862            1,
12863            runner.state.base_commit.clone(),
12864            reselected,
12865        )
12866        .await
12867        .unwrap();
12868        assert_eq!(
12869            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12870                .await
12871                .unwrap(),
12872            head
12873        );
12874        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12875        assert_eq!(
12876            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12877            head
12878        );
12879    }
12880
12881    #[tokio::test]
12882    #[cfg(unix)]
12883    async fn fallback_and_interrupted_restoration_require_coordination() {
12884        use std::os::fd::AsRawFd as _;
12885        ask_test_home();
12886        let tmp = tempfile::tempdir().unwrap();
12887        let mut runner = registered_runner(tmp.path()).await;
12888        runner.state.config.graph.timeout_resource_wait = 1;
12889        let winner = runner.state.candidates[0].clone();
12890        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12891        let leftover = winner.worktree.join("leftover");
12892        std::fs::write(&leftover, "untrusted edits").unwrap();
12893        let next = tmp.path().join("next-fixer");
12894        let mut failed = spec("first");
12895        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12896        let mut successor = spec("second");
12897        successor.command = vec![
12898            "sh".into(),
12899            "-c".into(),
12900            format!("touch '{}'; echo done", next.display()),
12901        ];
12902        let attempts = vec![
12903            (failed.clone(), "first".into()),
12904            (successor.clone(), "second".into()),
12905        ];
12906        let lock_path = runner.state.repo.join(".git/updater.lock");
12907        let external = std::fs::File::create(&lock_path).unwrap();
12908        assert_eq!(
12909            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12910            0
12911        );
12912        let artifacts = tmp.path().join("artifacts");
12913        let build = |spec, seat| SeatJob {
12914            spec,
12915            seat,
12916            cwd: winner.worktree.clone(),
12917            prompt: "fixture".into(),
12918            timeout: Duration::from_secs(10),
12919            allow_write: true,
12920            sessions: false,
12921            artifacts: artifacts.clone(),
12922            stem: "fix".into(),
12923            handover: None,
12924        };
12925        let (_, _, out) = runner
12926            .ask_fixer_among(
12927                attempts.clone(),
12928                "graft",
12929                None,
12930                Some((&winner.worktree, &head)),
12931                build,
12932            )
12933            .await;
12934        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12935        assert!(leftover.exists(), "contention must prevent clean");
12936        assert!(!next.exists());
12937        assert!(runner.state.handovers.is_empty());
12938        // Early interrupted-repair rollback uses the same protected route.
12939        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12940            from_run: "earlier".into(),
12941            generation: 1,
12942            mode: crate::recovery::Mode::VerifyRepair,
12943            reasons: Vec::new(),
12944            reviewed_head: None,
12945            rounds: Vec::new(),
12946            open_findings: Vec::new(),
12947            evidence: None,
12948            criteria_inherited: true,
12949            repair: crate::recovery::Repair::Started {
12950                before: head.clone(),
12951            },
12952        });
12953        assert!(
12954            runner
12955                .recovery_repair(&winner)
12956                .await
12957                .unwrap_err()
12958                .is::<crate::repo_lock::ResourceBlocked>()
12959        );
12960        assert!(matches!(
12961            runner.state.recovery.as_ref().unwrap().repair,
12962            crate::recovery::Repair::Started { .. }
12963        ));
12964        assert!(leftover.exists());
12965        drop(external);
12966        runner.recovery_repair(&winner).await.unwrap();
12967        assert!(!leftover.exists());
12968        assert!(matches!(
12969            runner.state.recovery.as_ref().unwrap().repair,
12970            crate::recovery::Repair::Done { .. }
12971        ));
12972        // A hard lock-open failure also leaves HEAD and untracked work intact.
12973        std::fs::remove_file(&lock_path).unwrap();
12974        std::fs::create_dir(&lock_path).unwrap();
12975        std::fs::write(&leftover, "untrusted edits").unwrap();
12976        let (_, _, out) = runner
12977            .ask_fixer_among(
12978                attempts,
12979                "graft",
12980                None,
12981                Some((&winner.worktree, &head)),
12982                build,
12983            )
12984            .await;
12985        assert!(matches!(out, AgentOutcome::Failed(_)));
12986        assert!(leftover.exists());
12987        assert!(!next.exists());
12988        assert_eq!(
12989            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12990            head
12991        );
12992    }
12993
12994    #[cfg(unix)]
12995    async fn registered_runner(root: &Path) -> Runner {
12996        ask_test_home();
12997        let repo = root.join("repo");
12998        std::fs::create_dir_all(&repo).unwrap();
12999        init_repo(&repo);
13000        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13001        let mut runner = runner_at(RunStatus::Prep);
13002        runner.state.repo = repo.clone();
13003        runner.state.config.coordination = crate::config::test_coordination();
13004        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13005        runner.state.config.graph.worktree_root = Some(root.join("wt"));
13006        runner.state.config.graph.advise = false;
13007        runner.roles.implementers = vec![spec("implement")];
13008        runner.roles.judges = vec![spec("judge")];
13009        runner.prep().await.unwrap();
13010        runner
13011    }
13012
13013    #[tokio::test]
13014    #[cfg(unix)]
13015    async fn parked_implementer_persists_completed_work() {
13016        for registered in [false, true] {
13017            let tmp = tempfile::tempdir().unwrap();
13018            let mut runner = registered_runner(tmp.path()).await;
13019            if !registered {
13020                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13021            }
13022            let marker = tmp.path().join("started");
13023            let release = tmp.path().join("release");
13024            let mut implementer = spec("implement");
13025            implementer.command = vec![
13026                "sh".into(),
13027                "-c".into(),
13028                format!(
13029                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13030                    marker.display(),
13031                    release.display()
13032                ),
13033            ];
13034            runner.roles.implementers = vec![implementer.clone()];
13035            runner.state.config.agents = vec![implementer];
13036            let pause = runner.pause.clone();
13037            let (result, ()) = tokio::join!(runner.implement(), async {
13038                tokio::time::timeout(Duration::from_secs(10), async {
13039                    while !marker.exists() {
13040                        tokio::time::sleep(Duration::from_millis(10)).await;
13041                    }
13042                })
13043                .await
13044                .unwrap();
13045                pause.park();
13046                std::fs::write(&release, "go").unwrap();
13047            });
13048            result.unwrap();
13049            let candidate = &runner.state.candidates[0];
13050            assert!(candidate.commits > 0, "{candidate:?}");
13051            assert!(candidate.summary.contains("saved-result"));
13052            assert_eq!(candidate.files, 1);
13053            let before = candidate.commits;
13054            runner.implement().await.unwrap();
13055            assert_eq!(runner.state.candidates[0].commits, before);
13056        }
13057    }
13058
13059    #[tokio::test]
13060    #[cfg(unix)]
13061    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13062        let tmp = tempfile::tempdir().unwrap();
13063        let mut runner = registered_runner(tmp.path()).await;
13064        let winner = runner.state.candidates[0].clone();
13065        let cache = tmp.path().join("cache");
13066        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13067        runner.state.config.graph.timeout_resource_wait = 0;
13068        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13069        let crate::cache::AcquireOutcome::Acquired(held) =
13070            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13071        else {
13072            panic!("cache must be free")
13073        };
13074        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13075            from_run: "earlier".into(),
13076            generation: 1,
13077            mode: crate::recovery::Mode::VerifyRepair,
13078            reasons: Vec::new(),
13079            reviewed_head: None,
13080            rounds: Vec::new(),
13081            open_findings: Vec::new(),
13082            evidence: None,
13083            criteria_inherited: true,
13084            repair: crate::recovery::Repair::NotStarted,
13085        });
13086        runner.recovery_repair(&winner).await.unwrap();
13087        assert!(matches!(
13088            runner.state.recovery.as_ref().unwrap().repair,
13089            crate::recovery::Repair::WaitingForRestore { .. }
13090        ));
13091        drop(held);
13092        runner.state.resource_block = None;
13093        runner.state.config.graph.timeout_resource_wait = 1;
13094        runner.recovery_repair(&winner).await.unwrap();
13095        assert!(matches!(
13096            runner.state.recovery.as_ref().unwrap().repair,
13097            crate::recovery::Repair::Done { .. }
13098        ));
13099        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13100    }
13101
13102    #[tokio::test]
13103    #[cfg(unix)]
13104    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13105        use std::os::fd::AsRawFd as _;
13106        for graft in [false, true] {
13107            let tmp = tempfile::tempdir().unwrap();
13108            let mut runner = registered_runner(tmp.path()).await;
13109            runner.state.config.graph.timeout_resource_wait = 1;
13110            let winner = runner.state.candidates[0].clone();
13111            let marker = tmp.path().join("fixer-started");
13112            let mut fixer = spec("fixer");
13113            fixer.command = vec![
13114                "sh".into(),
13115                "-c".into(),
13116                format!("echo started >> '{}'; echo done", marker.display()),
13117            ];
13118            runner.roles.fixer = Some(vec![fixer]);
13119            if graft {
13120                let mut loser = winner.clone();
13121                loser.label = 'B';
13122                loser.index = 1;
13123                runner.state.candidates.push(loser);
13124                runner.state.config.graph.graft_runner_up = true;
13125                runner.state.tally = Some(Tally {
13126                    first_choice: BTreeMap::new(),
13127                    borda: BTreeMap::new(),
13128                    winner: winner.label,
13129                    rankings: 1,
13130                    unanimous_initial: true,
13131                    deliberated: false,
13132                    changed_votes: 0,
13133                    unanimous_final: true,
13134                    tie_break: None,
13135                    judges: 1,
13136                    present: 1,
13137                    quorum: 1,
13138                    met_quorum: true,
13139                    uncontested: None,
13140                });
13141                runner.state.judgements.push(crate::run::Judgement {
13142                    judge: 1,
13143                    seat: "judge-1".into(),
13144                    agent: "judge".into(),
13145                    ranking: vec![winner.label, 'B'],
13146                    reasons: BTreeMap::new(),
13147                    confidence: None,
13148                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13149                    order: vec![0, 1],
13150                    failed: None,
13151                    duration_ms: 0,
13152                });
13153            } else {
13154                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13155                    from_run: "earlier".into(),
13156                    generation: 1,
13157                    mode: crate::recovery::Mode::VerifyRepair,
13158                    reasons: Vec::new(),
13159                    reviewed_head: None,
13160                    rounds: Vec::new(),
13161                    open_findings: Vec::new(),
13162                    evidence: None,
13163                    criteria_inherited: true,
13164                    repair: crate::recovery::Repair::NotStarted,
13165                });
13166            }
13167            let reader = crate::repo_lock::SeatLease::for_checkout(
13168                &winner.worktree,
13169                false,
13170                crate::config::test_coordination().as_ref(),
13171            )
13172            .await
13173            .unwrap();
13174            let pause = runner.pause.clone();
13175            let (result, ()) = tokio::join!(
13176                async {
13177                    if graft {
13178                        runner.graft().await
13179                    } else {
13180                        runner.recovery_repair(&winner).await
13181                    }
13182                },
13183                async {
13184                    tokio::time::sleep(Duration::from_millis(100)).await;
13185                    pause.park();
13186                }
13187            );
13188            result.unwrap();
13189            assert!(!marker.exists(), "a blocked seat must not spawn");
13190            // Exercise the persisted state, as a binary replacement/resume does.
13191            runner.state = RunState::load(&runner.state.id).unwrap();
13192            if graft {
13193                assert_eq!(
13194                    runner.state.graft.as_ref().unwrap().outcome,
13195                    GraftOutcome::WaitingForRestore
13196                );
13197            } else {
13198                assert!(matches!(
13199                    runner.state.recovery.as_ref().unwrap().repair,
13200                    crate::recovery::Repair::WaitingForRestore { .. }
13201                ));
13202            }
13203            drop(reader);
13204            runner.pause = Pause::new();
13205            runner.state.resource_block = None;
13206            // Deferred restoration must still respect an external lock owner.
13207            let external =
13208                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13209            assert_eq!(
13210                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13211                0
13212            );
13213            let leftover = winner.worktree.join("leftover");
13214            std::fs::write(&leftover, "earlier fallback edits").unwrap();
13215            let result = if graft {
13216                runner.graft().await
13217            } else {
13218                runner.recovery_repair(&winner).await
13219            };
13220            assert!(
13221                result
13222                    .unwrap_err()
13223                    .is::<crate::repo_lock::ResourceBlocked>()
13224            );
13225            assert!(leftover.exists());
13226            assert!(!marker.exists());
13227            drop(external);
13228            runner.state.resource_block = None;
13229            let result = if graft {
13230                runner.graft().await
13231            } else {
13232                runner.recovery_repair(&winner).await
13233            };
13234            result.unwrap();
13235            assert!(!leftover.exists(), "restore must precede retry");
13236            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13237            if graft {
13238                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13239            } else {
13240                assert!(matches!(
13241                    runner.state.recovery.as_ref().unwrap().repair,
13242                    crate::recovery::Repair::Done { .. }
13243                ));
13244            }
13245        }
13246    }
13247
13248    #[tokio::test]
13249    #[cfg(unix)]
13250    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13251        use std::os::fd::AsRawFd as _;
13252        let tmp = tempfile::tempdir().unwrap();
13253        let mut runner = registered_runner(tmp.path()).await;
13254        runner.state.config.graph.timeout_resource_wait = 1;
13255        let winner = runner.state.candidates[0].clone();
13256        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13257        runner.state.tally = Some(Tally {
13258            first_choice: BTreeMap::new(),
13259            borda: BTreeMap::new(),
13260            winner: winner.label,
13261            rankings: 0,
13262            unanimous_initial: false,
13263            deliberated: false,
13264            changed_votes: 0,
13265            unanimous_final: false,
13266            tie_break: None,
13267            judges: 0,
13268            present: 0,
13269            quorum: 0,
13270            met_quorum: true,
13271            uncontested: Some("fixture".into()),
13272        });
13273        runner.state.graft = Some(GraftRecord {
13274            pre: pre.clone(),
13275            post: None,
13276            lines: None,
13277            limit: None,
13278            agent: None,
13279            outcome: GraftOutcome::Running,
13280            requested: Vec::new(),
13281            adoption: None,
13282            adoption_error: None,
13283        });
13284        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13285        git::commit_all(&winner.worktree, "unfinished graft")
13286            .await
13287            .unwrap();
13288        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13289        let leftover = winner.worktree.join("leftover");
13290        std::fs::write(&leftover, "untrusted edits").unwrap();
13291        let path = runner.state.repo.join(".git/updater.lock");
13292        let external = std::fs::File::create(&path).unwrap();
13293        assert_eq!(
13294            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13295            0
13296        );
13297        assert!(
13298            runner
13299                .graft()
13300                .await
13301                .unwrap_err()
13302                .is::<crate::repo_lock::ResourceBlocked>()
13303        );
13304        assert!(matches!(
13305            runner.state.graft.as_ref().unwrap().outcome,
13306            GraftOutcome::Running
13307        ));
13308        assert_eq!(
13309            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13310            unfinished
13311        );
13312        assert!(leftover.exists());
13313        drop(external);
13314        std::fs::remove_file(&path).unwrap();
13315        std::fs::create_dir(&path).unwrap();
13316        assert!(runner.graft().await.is_err());
13317        assert!(matches!(
13318            runner.state.graft.as_ref().unwrap().outcome,
13319            GraftOutcome::Running
13320        ));
13321        assert_eq!(
13322            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13323            unfinished
13324        );
13325        assert!(leftover.exists());
13326        std::fs::remove_dir(&path).unwrap();
13327        runner.graft().await.unwrap();
13328        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13329        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13330        assert!(!leftover.exists());
13331    }
13332
13333    #[tokio::test]
13334    #[cfg(unix)]
13335    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13336        use std::os::fd::AsRawFd as _;
13337        let tmp = tempfile::tempdir().unwrap();
13338        let mut runner = registered_runner(tmp.path()).await;
13339        runner.state.prep_complete = Some(false);
13340        runner.state.config.graph.timeout_resource_wait = 1;
13341        runner.state.save().unwrap();
13342        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13343        assert_eq!(
13344            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13345            0
13346        );
13347        runner.execute().await.unwrap();
13348        let disk = RunState::load(&runner.state.id).unwrap();
13349        assert!(disk.driver_exited);
13350        assert_eq!(disk.driver_pid, None);
13351        assert_eq!(runner.state.status, RunStatus::Blocked);
13352        assert_eq!(
13353            runner.state.resource_block.as_ref().unwrap().kind,
13354            "repository coordination"
13355        );
13356        assert!(runner.state.prep_failure.is_none());
13357        assert!(runner.state.seats.is_empty());
13358        runner.pause.park();
13359        // Include headroom for Git subprocesses under parallel test load;
13360        // the repository contention deadline itself remains one second.
13361        tokio::time::timeout(Duration::from_secs(10), runner.execute())
13362            .await
13363            .unwrap()
13364            .unwrap();
13365        assert!(runner.state.parked);
13366        assert!(runner.state.prep_failure.is_none());
13367        let disk = RunState::load(&runner.state.id).unwrap();
13368        assert!(disk.driver_exited);
13369        assert_eq!(disk.driver_pid, None);
13370    }
13371
13372    #[tokio::test]
13373    #[cfg(unix)]
13374    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13375        use std::os::fd::AsRawFd as _;
13376        let tmp = tempfile::tempdir().unwrap();
13377        let mut runner = registered_runner(tmp.path()).await;
13378        runner.state.prep_complete = Some(false);
13379        runner.state.config.graph.timeout_resource_wait = 30;
13380        runner.state.save().unwrap();
13381        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13382        assert_eq!(
13383            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13384            0
13385        );
13386        assert!(
13387            tokio::time::timeout(Duration::from_millis(100), runner.execute())
13388                .await
13389                .is_err()
13390        );
13391        let disk = RunState::load(&runner.state.id).unwrap();
13392        assert!(disk.driver_exited);
13393        assert_eq!(disk.driver_pid, None);
13394        assert!(disk.status.resumable());
13395        assert!(runner.state.driver_exited);
13396        assert_eq!(runner.state.driver_pid, None);
13397        runner.state.save().unwrap();
13398        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13399    }
13400
13401    #[tokio::test]
13402    #[cfg(unix)]
13403    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13404        let tmp = tempfile::tempdir().unwrap();
13405        let mut runner = registered_runner(tmp.path()).await;
13406        let winner = runner.state.candidates[0].clone();
13407        let cache = tmp.path().join("cache");
13408        let started = tmp.path().join("pre-gate-started");
13409        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13410        runner.state.config.verify.gate =
13411            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13412        let home = crate::run::home();
13413        let owner =
13414            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13415        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13416            crate::cache::AcquireOutcome::Acquired(g) => g,
13417            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13418        };
13419        let mut gate = Box::pin(runner.run_gate(&winner));
13420        assert!(
13421            tokio::time::timeout(Duration::from_millis(150), &mut gate)
13422                .await
13423                .is_err()
13424        );
13425        assert!(!started.exists(), "pre-gate must wait for the cache too");
13426        // Simulate the wave that already owns the cache entering invoke.
13427        // It must obtain the editor reservation and finish without waiting
13428        // for the gate that is queued behind its cache.
13429        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13430        let editor = crate::repo_lock::SeatLease::checkout_with(
13431            &winner.worktree,
13432            true,
13433            crate::config::test_coordination().as_ref(),
13434            &wait,
13435        )
13436        .await
13437        .unwrap();
13438        drop(editor);
13439        guard.release();
13440        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13441            .await
13442            .unwrap()
13443            .unwrap();
13444        drop(gate);
13445        assert!(outcomes.iter().all(CommandOutcome::ok));
13446        assert!(started.exists());
13447    }
13448
13449    #[tokio::test]
13450    #[cfg(unix)]
13451    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13452        use std::os::fd::AsRawFd as _;
13453
13454        ask_test_home();
13455        let tmp = tempfile::tempdir().unwrap();
13456        let repo = tmp.path().join("repo");
13457        std::fs::create_dir_all(&repo).unwrap();
13458        init_repo(&repo);
13459        let common = repo.join(".git");
13460        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13461        let mut runner = runner_at(RunStatus::Prep);
13462        runner.state.repo = repo.clone();
13463        runner.state.config.coordination = crate::config::test_coordination();
13464        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13465        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13466        runner.state.config.graph.advise = false;
13467        runner.roles.implementers = vec![spec("implement")];
13468        runner.roles.judges = vec![spec("judge")];
13469        runner.prep().await.unwrap();
13470        let winner = runner.state.candidates[0].clone();
13471        // Emulate edits already left by a completed seat. The fixture CLI
13472        // only reports completion; it performs no repository operations.
13473        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13474        let completed = tmp.path().join("seat-completed");
13475        runner.roles.implementers[0].command = vec![
13476            "sh".into(),
13477            "-c".into(),
13478            format!("touch '{}'; echo completed", completed.display()),
13479        ];
13480        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13481        assert_eq!(
13482            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13483            0
13484        );
13485        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13486        let mut implement = Box::pin(runner.implement());
13487        tokio::select! {
13488            result = &mut implement => panic!("rescue must wait: {result:?}"),
13489            result = tokio::time::timeout(Duration::from_secs(10), async {
13490                while !completed.exists() {
13491                    tokio::time::sleep(Duration::from_millis(20)).await;
13492                }
13493            }) => result.unwrap(),
13494        }
13495        assert!(
13496            tokio::time::timeout(Duration::from_millis(200), &mut implement)
13497                .await
13498                .is_err()
13499        );
13500        assert_eq!(
13501            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13502            before
13503        );
13504        assert_eq!(
13505            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13506            0
13507        );
13508        tokio::time::timeout(Duration::from_secs(10), &mut implement)
13509            .await
13510            .unwrap()
13511            .unwrap();
13512        drop(implement);
13513        assert_ne!(
13514            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13515            before
13516        );
13517        assert!(git::is_clean(&winner.worktree).await.unwrap());
13518
13519        let started = tmp.path().join("pre-gate-started");
13520        let proceed = tmp.path().join("pre-gate-proceed");
13521        let checked = tmp.path().join("gate-ran");
13522        runner.state.config.verify.pre_gate = vec![format!(
13523            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13524            started.display(),
13525            proceed.display()
13526        )];
13527        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13528        assert_eq!(
13529            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13530            0
13531        );
13532        let mut gate = Box::pin(runner.run_gate(&winner));
13533        assert!(
13534            tokio::time::timeout(Duration::from_millis(200), &mut gate)
13535                .await
13536                .is_err()
13537        );
13538        assert!(!started.exists());
13539        assert!(!checked.exists());
13540        assert!(git::is_clean(&winner.worktree).await.unwrap());
13541        assert_eq!(
13542            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13543            0
13544        );
13545        tokio::select! {
13546            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13547            result = tokio::time::timeout(Duration::from_secs(10), async {
13548                while !started.exists() {
13549                    tokio::time::sleep(Duration::from_millis(20)).await;
13550                }
13551            }) => result.unwrap(),
13552        }
13553        assert_eq!(
13554            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13555            -1,
13556            "mechanical edits must hold the real lock"
13557        );
13558        std::fs::write(proceed, "ready").unwrap();
13559        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13560            .await
13561            .unwrap()
13562            .unwrap();
13563        drop(gate);
13564        assert!(outcomes.iter().all(CommandOutcome::ok));
13565        assert!(checked.exists());
13566        assert!(runner.state.pre_gate_commit.is_some());
13567        assert!(git::is_clean(&winner.worktree).await.unwrap());
13568        assert_eq!(
13569            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13570            0,
13571            "gate completion must release coordination"
13572        );
13573        drop(outsider);
13574        std::fs::remove_file(common.join("updater.lock")).unwrap();
13575        std::fs::create_dir(common.join("updater.lock")).unwrap();
13576        std::fs::remove_file(&started).unwrap();
13577        std::fs::remove_file(&checked).unwrap();
13578        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13579        let error = runner.run_gate(&winner).await.unwrap_err();
13580        assert!(
13581            error
13582                .to_string()
13583                .contains("acquire mandatory repository lock")
13584        );
13585        assert!(
13586            !started.exists(),
13587            "failed acquisition must prevent pre-gate"
13588        );
13589        assert!(!checked.exists(), "failed acquisition must prevent gate");
13590        assert!(git::is_clean(&winner.worktree).await.unwrap());
13591        assert_eq!(
13592            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13593            head
13594        );
13595    }
13596
13597    #[tokio::test]
13598    #[cfg(unix)]
13599    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13600        use std::os::fd::AsRawFd as _;
13601        use std::time::Duration;
13602
13603        ask_test_home();
13604        let tmp = tempfile::tempdir().unwrap();
13605        let repo = tmp.path().join("repo");
13606        std::fs::create_dir_all(&repo).unwrap();
13607        init_repo(&repo);
13608        let common = repo.join(".git");
13609        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13610        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13611        assert_eq!(
13612            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13613            0
13614        );
13615        let mut runner = runner_at(RunStatus::Prep);
13616        runner.state.repo = repo.clone();
13617        runner.state.config.coordination = crate::config::test_coordination();
13618        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13619        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13620        runner.state.config.graph.advise = true;
13621        runner.state.config.graph.advisors = 1;
13622        runner.roles.implementers = vec![spec("implement")];
13623        runner.roles.judges = vec![spec("judge")];
13624        let root = runner.state.worktree_root();
13625        let started = tmp.path().join("setup-started");
13626        let proceed = tmp.path().join("setup-proceed");
13627        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13628            run: Some(format!(
13629                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13630                started.display(),
13631                proceed.display()
13632            )),
13633            ..Default::default()
13634        }];
13635        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13636        let config = git::git(&repo, &["config", "--local", "--list"])
13637            .await
13638            .unwrap();
13639        let mut prep = Box::pin(runner.prep());
13640        assert!(
13641            tokio::time::timeout(Duration::from_millis(200), &mut prep)
13642                .await
13643                .is_err()
13644        );
13645        assert!(
13646            !root.exists(),
13647            "contention must prevent even checkout creation"
13648        );
13649        assert!(!started.exists(), "contention must prevent setup");
13650        assert_eq!(
13651            git::git(&repo, &["branch", "--list"]).await.unwrap(),
13652            branches
13653        );
13654        assert_eq!(
13655            git::git(&repo, &["config", "--local", "--list"])
13656                .await
13657                .unwrap(),
13658            config
13659        );
13660        drop(lock);
13661        tokio::select! {
13662            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13663            result = tokio::time::timeout(Duration::from_secs(10), async {
13664                while !started.exists() {
13665                    tokio::time::sleep(Duration::from_millis(20)).await;
13666                }
13667            }) => result.unwrap(),
13668        }
13669        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13670        assert_eq!(
13671            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13672            -1
13673        );
13674        assert!(
13675            tokio::time::timeout(
13676                Duration::from_millis(100),
13677                crate::repo_lock::SeatLease::for_checkout(
13678                    &repo,
13679                    true,
13680                    crate::config::test_coordination().as_ref()
13681                )
13682            )
13683            .await
13684            .is_err(),
13685            "preparation must also exclude local editors"
13686        );
13687        std::fs::write(proceed, "ready").unwrap();
13688        tokio::time::timeout(Duration::from_secs(10), &mut prep)
13689            .await
13690            .unwrap()
13691            .unwrap();
13692        drop(prep);
13693        assert_eq!(runner.state.prep_complete, Some(true));
13694        assert!(root.join("judge-1").exists());
13695        assert!(root.join("advisor-1").exists());
13696        assert_eq!(
13697            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13698            0
13699        );
13700    }
13701
13702    #[tokio::test]
13703    #[cfg(unix)]
13704    async fn preparation_lock_failure_prevents_repository_changes() {
13705        ask_test_home();
13706        let tmp = tempfile::tempdir().unwrap();
13707        let repo = tmp.path().join("repo");
13708        std::fs::create_dir_all(&repo).unwrap();
13709        init_repo(&repo);
13710        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13711        // A directory at the required lock path makes opening the real lock
13712        // fail, rather than merely exercising an invalid registration.
13713        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13714        let mut runner = runner_at(RunStatus::Prep);
13715        runner.state.repo = repo.clone();
13716        runner.state.config.coordination = crate::config::test_coordination();
13717        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13718        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13719        runner.roles.implementers = vec![spec("implement")];
13720        let error = runner.prep().await.unwrap_err();
13721        assert!(
13722            error
13723                .to_string()
13724                .contains("acquire mandatory repository lock")
13725        );
13726        assert!(!runner.state.worktree_root().exists());
13727        assert!(runner.state.candidates.is_empty());
13728        assert!(!repo.join(".git/config.worktree").exists());
13729        assert_eq!(
13730            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13731            "* main"
13732        );
13733    }
13734
13735    #[tokio::test]
13736    async fn preparation_records_creation_intent_before_git_can_fail() {
13737        ask_test_home();
13738        let tmp = tempfile::tempdir().unwrap();
13739        let repo = tmp.path().join("repo");
13740        std::fs::create_dir_all(&repo).unwrap();
13741        init_repo(&repo);
13742        let mut runner = runner_at(RunStatus::Prep);
13743        runner.state.repo = repo.clone();
13744        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13745        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13746        runner.state.config.blind.commit_msg_hook = false;
13747        runner.roles.implementers = vec![spec("one")];
13748        let label = blind::assign_labels(1, runner.state.seed)[0];
13749        let branch = runner.state.branch_for(label);
13750        // A conflicting branch makes Git refuse creation before a candidate
13751        // can be registered. The durable intent must still identify it.
13752        git::git(&repo, &["branch", &branch]).await.unwrap();
13753        assert!(runner.prep().await.is_err());
13754        let saved = RunState::load(&runner.state.id).unwrap();
13755        assert!(saved.candidates.is_empty());
13756        let event = saved
13757            .events
13758            .iter()
13759            .find(|e| e.node == "prep-intent")
13760            .unwrap();
13761        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13762        assert_eq!(intent["branch"], branch);
13763        assert_eq!(intent["start"], runner.state.base_commit);
13764        assert_eq!(
13765            intent["worktree"],
13766            runner
13767                .state
13768                .worktree_root()
13769                .join(format!("cand-{label}"))
13770                .to_string_lossy()
13771                .as_ref()
13772        );
13773        assert_eq!(intent["picks"], serde_json::json!([]));
13774    }
13775
13776    #[tokio::test]
13777    async fn invalid_pinned_sources_create_no_partial_worktrees() {
13778        ask_test_home();
13779        let tmp = tempfile::tempdir().unwrap();
13780        let repo = tmp.path().join("repo");
13781        std::fs::create_dir_all(&repo).unwrap();
13782        init_repo(&repo);
13783        let mut runner = runner_at(RunStatus::Prep);
13784        runner.state.repo = repo.clone();
13785        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13786        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13787        runner.state.sources.pick_commits.push(refs::Input {
13788            token: "missing".into(),
13789            sha: "0".repeat(40),
13790            provenance: refs::Provenance::ExplicitCli,
13791        });
13792        assert!(runner.prep().await.is_err());
13793        assert!(runner.state.candidates.is_empty());
13794        assert!(!runner.state.worktree_root().exists());
13795    }
13796
13797    #[tokio::test]
13798    async fn prose_and_legacy_inference_never_import_into_candidates() {
13799        ask_test_home();
13800        let tmp = tempfile::tempdir().unwrap();
13801        let repo = tmp.path().join("repo");
13802        std::fs::create_dir_all(&repo).unwrap();
13803        init_repo(&repo);
13804        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13805        git::git(&repo, &["checkout", "-b", "magi/history/A"])
13806            .await
13807            .unwrap();
13808        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13809        git::git(&repo, &["add", "."]).await.unwrap();
13810        git::git(&repo, &["commit", "-m", "historical implementation"])
13811            .await
13812            .unwrap();
13813        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13814        git::git(&repo, &["checkout", "main"]).await.unwrap();
13815        for instruction in [
13816            format!("Example: magi/history/A at {source}"),
13817            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13818            format!("Never import magi/history/A or {source}"),
13819            format!("Conversation answer: I mentioned {source} only as background"),
13820        ] {
13821            let mut runner = runner_at(RunStatus::Prep);
13822            runner.state.repo = repo.clone();
13823            runner.state.base_commit = base.clone();
13824            runner.state.instruction = instruction.clone();
13825            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13826            runner.state.config.blind.commit_msg_hook = false;
13827            runner.state.seeds = vec![refs::Seed {
13828                token: "magi/history/A".into(),
13829                sha: source.clone(),
13830                kind: refs::SeedKind::Unmerged,
13831                branch: true,
13832                detail: String::new(),
13833            }];
13834            runner.roles.implementers = vec![spec("one")];
13835            assert_eq!(seeded_instruction(&runner.state), instruction);
13836            // Model an old record with inferred seeds but no structured input field.
13837            let mut legacy = serde_json::to_value(&runner.state).unwrap();
13838            legacy.as_object_mut().unwrap().remove("sources");
13839            runner.state = serde_json::from_value(legacy).unwrap();
13840            assert_eq!(runner.state.sources, refs::Sources::default());
13841            runner.state.save().unwrap();
13842            runner.state = RunState::load(&runner.state.id).unwrap();
13843            runner.prep().await.unwrap();
13844            for candidate in &runner.state.candidates {
13845                assert_eq!(
13846                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13847                    base
13848                );
13849                assert!(!candidate.worktree.join("unwanted").exists());
13850            }
13851        }
13852    }
13853
13854    #[tokio::test]
13855    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13856        ask_test_home();
13857        let tmp = tempfile::tempdir().unwrap();
13858        let repo = tmp.path().join("repo");
13859        std::fs::create_dir_all(&repo).unwrap();
13860        init_repo(&repo);
13861        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13862        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13863        std::fs::write(repo.join("referenced"), "required commit").unwrap();
13864        git::git(&repo, &["add", "."]).await.unwrap();
13865        git::git(&repo, &["commit", "-m", "required seed"])
13866            .await
13867            .unwrap();
13868        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13869        git::git(&repo, &["checkout", "main"]).await.unwrap();
13870        let mut runner = runner_at(RunStatus::Prep);
13871        runner.state.repo = repo.clone();
13872        runner.state.base_commit = base.clone();
13873        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13874        runner.state.config.graph.advise = false;
13875        runner.state.config.blind.commit_msg_hook = false;
13876        runner.state.sources =
13877            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13878                .await
13879                .unwrap();
13880        runner.roles.implementers = vec![spec("one"), spec("two")];
13881        runner.roles.judges = vec![spec("judge")];
13882        let label = blind::assign_labels(2, runner.state.seed)[0];
13883        let branch = runner.state.branch_for(label);
13884        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13885        git::worktree_add_branch(&repo, &worktree, &branch, &base)
13886            .await
13887            .unwrap();
13888        runner.state.candidates.push(Candidate {
13889            index: 0,
13890            label,
13891            agent: "one".into(),
13892            branch,
13893            worktree,
13894            summary: String::new(),
13895            stat: String::new(),
13896            files: 0,
13897            commits: 0,
13898            empty: false,
13899            failed: None,
13900            verified_noop: None,
13901            duration_ms: 0,
13902            folded: false,
13903        });
13904        runner.state.save().unwrap();
13905        // Reload the checkpoint immediately before its first cherry-pick.
13906        runner.state = RunState::load(&runner.state.id).unwrap();
13907        assert_eq!(runner.state.prep_complete, Some(false));
13908        runner.prep().await.unwrap();
13909        assert_eq!(runner.state.prep_complete, Some(true));
13910        assert_eq!(runner.state.candidates.len(), 2);
13911        for candidate in &runner.state.candidates {
13912            assert_eq!(
13913                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13914                git::tree_of(&repo, &seed).await.unwrap()
13915            );
13916            assert_eq!(
13917                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13918                "required commit"
13919            );
13920        }
13921        assert!(runner.state.worktree_root().join("judge-1").exists());
13922        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13923        runner.prep().await.unwrap();
13924        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13925
13926        runner.state.prep_complete = Some(false);
13927        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13928            run: Some(format!(
13929                "test -f '{}'",
13930                tmp.path().join("dependency-ready").display()
13931            )),
13932            ..Default::default()
13933        }];
13934        assert!(runner.prep().await.is_err());
13935        assert!(runner.state.prep_failure.is_some());
13936        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13937        runner.state = RunState::load(&runner.state.id).unwrap();
13938        assert!(
13939            runner.prep().await.is_err(),
13940            "correcting the cause alone must not auto-retry"
13941        );
13942        let failed_at = runner.state.events.last().unwrap().at;
13943        assert!(
13944            runner.retry_preparation(failed_at).is_err(),
13945            "old approvals cannot unlock the hold"
13946        );
13947        let mut task = crate::queue::Task::new(
13948            "retry".into(),
13949            "retry setup".into(),
13950            repo.clone(),
13951            crate::queue::Source::Human,
13952        );
13953        task.start(runner.state.id.clone());
13954        task.hold_machine(Some("setup failed".into()));
13955        task.release_operator("cli", None);
13956        runner
13957            .retry_preparation(task.resumption.as_ref().unwrap().at)
13958            .unwrap();
13959        runner.prep().await.unwrap();
13960        assert!(runner.state.prep_failure.is_none());
13961        assert_eq!(runner.state.prep_complete, Some(true));
13962        assert_eq!(runner.state.candidates.len(), 2);
13963        assert!(runner.state.worktree_root().join("judge-1").exists());
13964        for candidate in &runner.state.candidates {
13965            assert_eq!(
13966                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13967                git::tree_of(&repo, &seed).await.unwrap()
13968            );
13969        }
13970    }
13971
13972    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13973        let mut heads = Vec::new();
13974        for candidate in candidates {
13975            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13976        }
13977        heads
13978    }
13979
13980    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13981        ask_test_home();
13982        let tmp = tempfile::tempdir().unwrap();
13983        let repo = tmp.path().join("repo");
13984        std::fs::create_dir(&repo).unwrap();
13985        init_repo(&repo);
13986        let mut runner = runner_at(RunStatus::Reviewing);
13987        runner.state.repo = repo.clone();
13988        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13989        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13990        for (index, label) in ['A', 'B'].into_iter().enumerate() {
13991            let branch = format!("candidate-{label}");
13992            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13993            git::worktree_add_branch(&repo, &worktree, &branch, "main")
13994                .await
13995                .unwrap();
13996            if label == 'B' {
13997                std::fs::write(worktree.join("change"), "winner").unwrap();
13998                git::rescue_commit(&worktree, "candidate change")
13999                    .await
14000                    .unwrap();
14001            }
14002            runner.state.candidates.push(Candidate {
14003                index,
14004                label,
14005                agent: "alpha".into(),
14006                branch,
14007                worktree,
14008                summary: "completed reply".into(),
14009                stat: String::new(),
14010                files: usize::from(label == 'B'),
14011                commits: usize::from(label == 'B'),
14012                empty: label == 'A',
14013                failed: None,
14014                verified_noop: None,
14015                duration_ms: 1,
14016                folded: false,
14017            });
14018        }
14019        runner.state.judge_skipped = true;
14020        runner.state.tally = Some(Tally {
14021            first_choice: BTreeMap::new(),
14022            borda: BTreeMap::new(),
14023            winner: 'B',
14024            rankings: 0,
14025            unanimous_initial: false,
14026            deliberated: false,
14027            changed_votes: 0,
14028            unanimous_final: false,
14029            tie_break: None,
14030            judges: 0,
14031            present: 0,
14032            quorum: 0,
14033            met_quorum: true,
14034            uncontested: Some("only B produced a change".into()),
14035        });
14036        runner.state.save().unwrap();
14037        (tmp, runner)
14038    }
14039
14040    #[tokio::test]
14041    #[cfg(unix)]
14042    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14043        for (solo, failed_checkpoint) in
14044            [(true, false), (false, false), (true, true), (false, true)]
14045        {
14046            let (_tmp, mut runner) = preservation_resume_fixture().await;
14047            if solo {
14048                runner.state.candidates.retain(|c| c.label == 'A');
14049            }
14050            runner.state.candidates[0].empty = false;
14051            runner.state.candidates[0].summary.clear();
14052            runner.state.tally = None;
14053            runner.state.judge_skipped = false;
14054            runner.state.status = RunStatus::Implementing;
14055            runner.state.config.disk.min_free_bytes = u64::MAX;
14056            let mut implementer = spec("alpha");
14057            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14058            runner.roles.implementers = vec![implementer];
14059            let patch = runner.state.dir().join("artifacts/cand-A.patch");
14060            if failed_checkpoint {
14061                std::fs::create_dir_all(&patch).unwrap();
14062            }
14063            runner.state.save().unwrap();
14064            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14065            runner.state = RunState::load(&runner.state.id).unwrap();
14066            assert_eq!(runner.state.status, RunStatus::Blocked);
14067            assert!(runner.state.interrupted_implementations.contains(&'A'));
14068            assert_eq!(runner.state.candidates[0].commits, 0);
14069            assert!(!runner.state.candidates[0].empty);
14070            assert!(runner.state.candidates[0].failed.is_none());
14071            if failed_checkpoint {
14072                assert!(runner.state.candidate_preservation.contains_key(&'A'));
14073                std::fs::remove_dir(patch).unwrap();
14074            }
14075            runner.state.resource_block = None;
14076            runner.state.config.disk.min_free_bytes = 0;
14077            runner.roles.implementers[0].command = vec![
14078                "sh".into(),
14079                "-c".into(),
14080                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14081            ];
14082            runner.implement().await.unwrap();
14083            assert!(runner.state.interrupted_implementations.is_empty());
14084            assert!(runner.state.candidate_preservation.is_empty());
14085            assert_eq!(runner.state.candidates[0].commits, 1);
14086            assert!(!runner.state.candidates[0].empty);
14087            assert!(
14088                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14089                    .unwrap()
14090                    .contains("retried empty interrupted seat")
14091            );
14092            if !solo {
14093                assert_eq!(runner.state.candidates[1].commits, 1);
14094            }
14095        }
14096    }
14097
14098    #[tokio::test]
14099    #[cfg(unix)]
14100    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14101        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14102            let (_tmp, mut runner) = preservation_resume_fixture().await;
14103            runner.state.candidates.retain(|c| c.label == 'A');
14104            runner.state.candidates[0].empty = false;
14105            runner.state.candidates[0].summary.clear();
14106            runner.state.tally = None;
14107            runner.state.judge_skipped = false;
14108            runner.state.status = RunStatus::Implementing;
14109            // Force the real post-spawn watchdog, independently of available
14110            // disk space. Dirty/committed work predates its first measurement.
14111            runner.state.config.disk.min_free_bytes = u64::MAX;
14112            let wt = runner.state.candidates[0].worktree.clone();
14113            std::fs::write(
14114                wt.join("interrupted-source"),
14115                "retain implementation work\n",
14116            )
14117            .unwrap();
14118            if committed {
14119                git::rescue_commit(&wt, "implementation checkpoint")
14120                    .await
14121                    .unwrap();
14122            }
14123            let lock = git::git(
14124                &wt,
14125                &[
14126                    "rev-parse",
14127                    "--path-format=absolute",
14128                    "--git-path",
14129                    "index.lock",
14130                ],
14131            )
14132            .await
14133            .unwrap();
14134            if block_rescue {
14135                std::fs::write(&lock, "fixture-owned lock").unwrap();
14136            }
14137            let mut implementer = spec("alpha");
14138            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14139            runner.roles.implementers = vec![implementer];
14140            runner.state.save().unwrap();
14141            let result = runner.implement().await;
14142            assert_eq!(result.is_err(), block_rescue);
14143            assert_eq!(
14144                runner.state.resource_block.as_ref().unwrap().kind,
14145                "disk pressure"
14146            );
14147            assert!(
14148                runner
14149                    .state
14150                    .events
14151                    .iter()
14152                    .any(|e| e.message.contains("agent execution interrupted"))
14153            );
14154            assert!(
14155                runner
14156                    .state
14157                    .dir()
14158                    .join("artifacts/candidate-A-recovery/staged.patch")
14159                    .exists()
14160            );
14161            if block_rescue {
14162                let saved = RunState::load(&runner.state.id).unwrap();
14163                assert!(saved.candidate_preservation.contains_key(&'A'));
14164                assert!(!saved.candidates[0].empty);
14165                assert_eq!(
14166                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14167                    "retain implementation work\n"
14168                );
14169                std::fs::remove_file(lock).unwrap();
14170                runner.state = saved;
14171            } else {
14172                assert_eq!(runner.state.status, RunStatus::Blocked);
14173                assert_eq!(runner.state.candidates[0].commits, 1);
14174            }
14175            // Recovery must save the candidate before deciding whether another
14176            // implementation is needed. No implementer is available on retry.
14177            runner.roles.implementers.clear();
14178            runner.state.config.disk.min_free_bytes = 0;
14179            runner.state.resource_block = None;
14180            runner.implement().await.unwrap();
14181            assert!(runner.state.candidate_preservation.is_empty());
14182            assert_eq!(runner.state.candidates[0].commits, 1);
14183            assert!(!runner.state.candidates[0].empty);
14184            assert!(git::is_clean(&wt).await.unwrap());
14185            assert!(
14186                crate::run::read_artifact(&runner.state, "cand-A.patch")
14187                    .unwrap()
14188                    .contains("retain implementation work")
14189            );
14190        }
14191    }
14192
14193    #[tokio::test]
14194    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14195        let (_tmp, mut runner) = preservation_resume_fixture().await;
14196        // Schema 15 already confirmed the unfolded no-change result.
14197        runner.implement().await.unwrap();
14198        assert!(runner.state.candidate_preservation.is_empty());
14199        assert!(
14200            !runner
14201                .state
14202                .dir()
14203                .join("artifacts/candidate-A-recovery")
14204                .exists()
14205        );
14206        runner.fold_losers().await.unwrap();
14207        assert!(!runner.state.candidates[0].worktree.exists());
14208        for schema in [14, crate::run::SCHEMA] {
14209            runner.state.schema = schema;
14210            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14211            runner.state.save().unwrap();
14212            runner.state = RunState::load(&runner.state.id).unwrap();
14213            runner.implement().await.unwrap();
14214            assert!(runner.state.candidate_preservation.is_empty());
14215            assert!(runner.state.candidates[0].folded);
14216            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14217        }
14218    }
14219
14220    #[tokio::test]
14221    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14222        let (_tmp, mut runner) = preservation_resume_fixture().await;
14223        let wt = runner.state.candidates[0].worktree.clone();
14224        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14225        runner.state.schema = 14;
14226        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14227        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14228        let tally = serde_json::to_value(&runner.state.tally).unwrap();
14229        for _ in 0..2 {
14230            assert!(runner.implement().await.is_err());
14231            assert_eq!(runner.state.status, RunStatus::Blocked);
14232            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14233            assert!(
14234                runner.state.candidates[0].empty,
14235                "old verdict must remain unusable"
14236            );
14237            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14238            assert!(runner.state.judge_skipped);
14239            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14240            assert!(
14241                crate::run::read_artifact(&runner.state, "cand-A.patch")
14242                    .unwrap()
14243                    .contains("owned edits")
14244            );
14245            assert!(wt.exists());
14246            assert_eq!(
14247                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14248                    .await
14249                    .unwrap(),
14250                1
14251            );
14252            runner.state = RunState::load(&runner.state.id).unwrap();
14253        }
14254    }
14255
14256    #[tokio::test]
14257    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14258        let (_tmp, mut runner) = preservation_resume_fixture().await;
14259        let wt = runner.state.candidates[0].worktree.clone();
14260        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14261        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14262        let index_lock = PathBuf::from(
14263            git::git(
14264                &wt,
14265                &[
14266                    "rev-parse",
14267                    "--path-format=absolute",
14268                    "--git-path",
14269                    "index.lock",
14270                ],
14271            )
14272            .await
14273            .unwrap(),
14274        );
14275        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14276        runner.state.schema = 14;
14277        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14278        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14279        assert!(runner.implement().await.is_err());
14280        assert!(runner.after_implement().is_err());
14281        assert!(runner.fold_losers().await.is_err());
14282        assert_eq!(
14283            std::fs::read_to_string(wt.join("recovered")).unwrap(),
14284            "failed seat edits"
14285        );
14286        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14287        runner.state = RunState::load(&runner.state.id).unwrap();
14288        std::fs::remove_file(index_lock).unwrap();
14289        // Recovery finds changes that the old uncontested result never judged.
14290        assert!(runner.implement().await.is_err());
14291        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14292        assert!(runner.state.candidates[0].empty);
14293        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14294        assert!(
14295            crate::run::read_artifact(&runner.state, "cand-A.patch")
14296                .unwrap()
14297                .contains("failed seat edits")
14298        );
14299        assert!(wt.exists());
14300    }
14301
14302    #[tokio::test]
14303    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14304        let (_tmp, mut runner) = preservation_resume_fixture().await;
14305        let repo = runner.state.repo.clone();
14306        let common = git::git(
14307            &repo,
14308            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14309        )
14310        .await
14311        .unwrap();
14312        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14313        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14314        assert!(runner.fold_losers().await.is_err());
14315        assert!(!runner.state.candidates[0].worktree.exists());
14316        assert!(!runner.state.candidates[0].folded);
14317        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14318        runner.state = RunState::load(&runner.state.id).unwrap();
14319        std::fs::remove_file(branch_lock).unwrap();
14320        let judge = runner.state.worktree_root().join("judge-1");
14321        git::worktree_add_detached(&repo, &judge, "main")
14322            .await
14323            .unwrap();
14324        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14325        runner.roles.judges = vec![spec("judge")];
14326        assert!(runner.fold_losers().await.is_err());
14327        runner.state = RunState::load(&runner.state.id).unwrap();
14328        assert!(
14329            runner.state.candidates[0].folded,
14330            "candidate progress saved before judge failure"
14331        );
14332        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14333        assert!(judge.join("uncommitted").exists());
14334        std::fs::remove_file(judge.join("uncommitted")).unwrap();
14335        runner.fold_losers().await.unwrap();
14336        assert!(!judge.exists());
14337        runner.fold_losers().await.unwrap();
14338    }
14339
14340    #[tokio::test]
14341    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14342        let (_tmp, mut runner) = preservation_resume_fixture().await;
14343        runner.state.candidates.retain(|c| c.label == 'B');
14344        runner.state.tally = None;
14345        runner.state.judge_skipped = false;
14346        runner.state.prep_complete = Some(true);
14347        runner.state.config.graph.advise = false;
14348        runner.state.config.graph.selected_implementer = Some("alpha".into());
14349        let wt = runner.state.candidates[0].worktree.clone();
14350        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14351        runner
14352            .state
14353            .candidate_preservation
14354            .insert('B', "awaiting persistence".into());
14355        let index_lock = git::git(
14356            &wt,
14357            &[
14358                "rev-parse",
14359                "--path-format=absolute",
14360                "--git-path",
14361                "index.lock",
14362            ],
14363        )
14364        .await
14365        .unwrap();
14366        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14367        runner.state.save().unwrap();
14368        assert!(runner.execute_graph().await.is_err());
14369        assert_eq!(runner.state.status, RunStatus::Blocked);
14370        assert!(runner.state.candidate_preservation.contains_key(&'B'));
14371        std::fs::remove_file(index_lock).unwrap();
14372        // Park at the next graph boundary: this tests the real selected-seat
14373        // hold guard without dispatching reviewers or publishing anything.
14374        for _ in 0..2 {
14375            runner.state = RunState::load(&runner.state.id).unwrap();
14376            runner.pause.park();
14377            runner.execute_graph().await.unwrap();
14378            assert_eq!(runner.state.status, RunStatus::Implementing);
14379            assert!(runner.state.candidate_preservation.is_empty());
14380            assert_eq!(runner.state.candidates[0].commits, 2);
14381            assert!(runner.state.tally.is_none());
14382        }
14383        runner.implement().await.unwrap();
14384        assert_eq!(runner.state.status, RunStatus::Judging);
14385        // A genuine selected-implementer hold must still require release.
14386        runner.state.status = RunStatus::Blocked;
14387        runner.state.candidates[0].failed = Some("implementation failed".into());
14388        runner.state.save().unwrap();
14389        let error = runner.execute_graph().await.unwrap_err();
14390        assert!(
14391            error
14392                .to_string()
14393                .contains("selected implementation is held")
14394        );
14395    }
14396
14397    #[tokio::test]
14398    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14399        let (_tmp, mut runner) = preservation_resume_fixture().await;
14400        let repo = runner.state.repo.clone();
14401        let candidate = runner.state.candidates[0].clone();
14402        // Remove only the disposable fixture directory, leaving Git's linked
14403        // worktree registration and branch intact, as in a vanished checkout.
14404        let registered = git::worktree_holding(&repo, &candidate.branch)
14405            .await
14406            .unwrap()
14407            .unwrap();
14408        assert_eq!(
14409            registered.canonicalize().unwrap(),
14410            candidate.worktree.canonicalize().unwrap()
14411        );
14412        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14413        assert_eq!(
14414            git::worktree_holding(&repo, &candidate.branch)
14415                .await
14416                .unwrap(),
14417            Some(registered)
14418        );
14419        assert!(
14420            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14421            "the stale registration initially keeps the branch checked out"
14422        );
14423        runner.fold_losers().await.unwrap();
14424        assert!(runner.state.candidates[0].folded);
14425        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14426        assert_eq!(
14427            git::worktree_holding(&repo, &candidate.branch)
14428                .await
14429                .unwrap(),
14430            None
14431        );
14432        runner.state = RunState::load(&runner.state.id).unwrap();
14433        runner.fold_losers().await.unwrap();
14434    }
14435
14436    #[tokio::test]
14437    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14438        let (_tmp, mut runner) = preservation_resume_fixture().await;
14439        let repo = runner.state.repo.clone();
14440        let candidate = runner.state.candidates[0].clone();
14441        let winner = runner.state.candidates[1].clone();
14442        let registered = git::worktree_holding(&repo, &candidate.branch)
14443            .await
14444            .unwrap()
14445            .unwrap();
14446        assert_eq!(
14447            registered.canonicalize().unwrap(),
14448            candidate.worktree.canonicalize().unwrap()
14449        );
14450        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14451        assert_eq!(
14452            git::worktree_holding(&repo, &candidate.branch)
14453                .await
14454                .unwrap(),
14455            Some(registered)
14456        );
14457        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14458
14459        for _ in 0..2 {
14460            fold_run(&mut runner.state, false, &crate::run::home())
14461                .await
14462                .unwrap();
14463            runner.state = RunState::load(&runner.state.id).unwrap();
14464            assert!(runner.state.candidates[0].folded);
14465            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14466            assert_eq!(
14467                git::worktree_holding(&repo, &candidate.branch)
14468                    .await
14469                    .unwrap(),
14470                None
14471            );
14472            assert!(!runner.state.candidates[1].folded);
14473            assert!(winner.worktree.exists());
14474            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14475        }
14476    }
14477
14478    #[tokio::test]
14479    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14480        ask_test_home();
14481        let tmp = tempfile::tempdir().unwrap();
14482        let repo = tmp.path().join("repo");
14483        std::fs::create_dir(&repo).unwrap();
14484        init_repo(&repo);
14485        let mut runner = runner_at(RunStatus::Implementing);
14486        runner.state.repo = repo.clone();
14487        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14488        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14489        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14490            let branch = format!("candidate-{label}");
14491            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14492            git::worktree_add_branch(&repo, &wt, &branch, "main")
14493                .await
14494                .unwrap();
14495            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14496            git::git(&wt, &["add", "README.md"]).await.unwrap();
14497            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14498            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14499            runner.state.candidates.push(Candidate {
14500                index,
14501                label,
14502                agent: "alpha".to_owned(),
14503                branch,
14504                worktree: wt,
14505                summary: "completed implementation".to_owned(),
14506                stat: String::new(),
14507                files: 0,
14508                commits: 0,
14509                empty: false,
14510                failed: None,
14511                verified_noop: None,
14512                duration_ms: 42,
14513                folded: false,
14514            });
14515            runner
14516                .state
14517                .candidate_preservation
14518                .insert(label, "awaiting persistence".to_owned());
14519        }
14520        let wt = runner.state.candidates[0].worktree.clone();
14521        let index_lock = git::git(
14522            &wt,
14523            &[
14524                "rev-parse",
14525                "--path-format=absolute",
14526                "--git-path",
14527                "index.lock",
14528            ],
14529        )
14530        .await
14531        .unwrap();
14532        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14533        runner.state.save().unwrap();
14534        assert!(runner.persist_candidates().await.is_err());
14535        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14536        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14537        assert_eq!(
14538            runner.state.candidates[1].commits, 1,
14539            "other results are still collected"
14540        );
14541        assert!(
14542            !runner.state.candidates[0].empty,
14543            "failed rescue is not no change"
14544        );
14545        assert_eq!(
14546            runner.state.candidates[0].summary,
14547            "completed implementation"
14548        );
14549        assert!(runner.after_implement().is_err());
14550        let home = tmp.path().join("fold-home");
14551        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14552        assert!(wt.join("new-file").exists());
14553        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14554        assert!(recovery.join("staged.patch").exists());
14555        assert!(recovery.join("unstaged.patch").exists());
14556        assert_eq!(
14557            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14558            "untracked edits\n"
14559        );
14560        runner.state.begin_pass();
14561        runner.state.end_pass_parked(ParkCause::Upgrade {
14562            from: "1.0".into(),
14563            to: Some("1.1".into()),
14564        });
14565        runner.state.save().unwrap();
14566        runner.state = RunState::load(&runner.state.id).unwrap();
14567        assert!(matches!(
14568            runner.state.passes.last().unwrap().end,
14569            Some(crate::run::PassEnd::Parked {
14570                cause: ParkCause::Upgrade { .. }
14571            })
14572        ));
14573        std::fs::remove_file(index_lock).unwrap();
14574        // No implementer is configured: a persistence retry cannot dispatch a seat.
14575        runner.implement().await.unwrap();
14576        assert!(runner.state.candidate_preservation.is_empty());
14577        assert!(
14578            runner
14579                .state
14580                .candidates
14581                .iter()
14582                .all(|c| c.commits == 1 && !c.empty)
14583        );
14584        assert!(git::is_clean(&wt).await.unwrap());
14585        assert!(
14586            crate::run::read_artifact(&runner.state, "cand-A.patch")
14587                .unwrap()
14588                .contains("later tracked edits")
14589        );
14590    }
14591
14592    #[tokio::test]
14593    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14594        ask_test_home();
14595        let tmp = tempfile::tempdir().unwrap();
14596        let repo = tmp.path().join("repo");
14597        std::fs::create_dir(&repo).unwrap();
14598        init_repo(&repo);
14599        let wt = tmp.path().join("candidate");
14600        git::worktree_add_branch(&repo, &wt, "candidate", "main")
14601            .await
14602            .unwrap();
14603        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14604        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14605        assert!(wt.join("uncommitted").exists());
14606        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14607        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14608        let mut runner = runner_at(RunStatus::Implementing);
14609        runner.state.repo = repo.clone();
14610        runner.state.base_commit = "missing-base".to_owned();
14611        runner.state.candidates.push(Candidate {
14612            index: 0,
14613            label: 'A',
14614            agent: "alpha".into(),
14615            branch: "candidate".into(),
14616            worktree: wt.clone(),
14617            summary: "completed".into(),
14618            stat: String::new(),
14619            files: 0,
14620            commits: 0,
14621            empty: false,
14622            failed: None,
14623            verified_noop: None,
14624            duration_ms: 1,
14625            folded: false,
14626        });
14627        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14628        assert!(
14629            fold_run(&mut runner.state, true, &tmp.path().join("home"))
14630                .await
14631                .is_err()
14632        );
14633        assert!(!runner.state.candidates[0].folded);
14634        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14635        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14636        runner
14637            .state
14638            .candidate_preservation
14639            .insert('A', "pending".into());
14640        assert!(runner.persist_candidates().await.is_err());
14641        assert!(!runner.state.candidates[0].empty);
14642        assert!(!runner.state.candidates[0].folded);
14643        assert!(wt.exists());
14644        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14645        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14646        std::fs::create_dir_all(&patch_path).unwrap();
14647        assert!(
14648            runner.persist_candidates().await.is_err(),
14649            "artifact failure stays pending"
14650        );
14651        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14652        assert!(!runner.state.candidates[0].empty);
14653        std::fs::remove_dir(patch_path).unwrap();
14654        runner.persist_candidates().await.unwrap();
14655        assert!(
14656            runner.state.candidates[0].empty,
14657            "only successful inspection confirms no change"
14658        );
14659    }
14660
14661    fn init_repo(dir: &Path) {
14662        let run = |args: &[&str]| {
14663            let out = std::process::Command::new("git")
14664                .args(args)
14665                .current_dir(dir)
14666                .quiet()
14667                .output()
14668                .expect("spawn git");
14669            assert!(
14670                out.status.success(),
14671                "git {args:?} failed: {}",
14672                String::from_utf8_lossy(&out.stderr)
14673            );
14674        };
14675        run(&["init", "-b", "main"]);
14676        run(&["config", "user.name", "magi test"]);
14677        run(&["config", "user.email", "magi@example.com"]);
14678        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14679        run(&["add", "-A"]);
14680        run(&["commit", "-m", "init"]);
14681    }
14682
14683    // `settle_questions` is what closes the ghost the phone showed: a run's
14684    // seat asked something, the run then ended, and nothing was left to
14685    // abandon the question it left `open`. `HOME` is a process-wide
14686    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
14687    // first time it runs in the binary — every test below still reaches the
14688    // same directory whichever call won, and each gets its own run id from
14689    // `RunState::new`, so they never collide there.
14690    fn ask_test_home() {
14691        crate::run::pin_test_home();
14692    }
14693
14694    /// A minimal, git-free `Runner` at a given status — `settle_questions`
14695    /// reads nothing else off it.
14696    fn runner_at(status: RunStatus) -> Runner {
14697        let mut state = RunState::new(
14698            PathBuf::from("/nonexistent/repo"),
14699            "main".to_owned(),
14700            "deadbeef".to_owned(),
14701            "task".to_owned(),
14702            Config::default(),
14703        );
14704        // These fixtures share one home and save concurrently. Short clock
14705        // IDs can collide, making an ownership test read another test's run.
14706        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14707        state.id = format!(
14708            "20000101-000000-{:04x}",
14709            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14710        );
14711        state.status = status;
14712        Runner {
14713            state,
14714            roles: ResolvedRoles {
14715                implementers: Vec::new(),
14716                judges: Vec::new(),
14717                reviewers: Vec::new(),
14718                fixer: None,
14719                conductor: conductor(),
14720                implementer_roster: Vec::new(),
14721                judge_roster: Vec::new(),
14722                reviewer_roster: Vec::new(),
14723            },
14724            sem: Arc::new(Semaphore::new(1)),
14725            pause: Pause::new(),
14726            interrupt: Pause::new(),
14727        }
14728    }
14729
14730    #[tokio::test]
14731    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
14732        crate::run::pin_test_home();
14733        let mut runner = runner_at(RunStatus::Gating);
14734        runner.state.save().unwrap();
14735        let id = runner.state.id.clone();
14736        let passes = runner.state.passes.len();
14737        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
14738        let mut claiming = Box::pin(runner.claim_driver());
14739        std::future::poll_fn(|cx| {
14740            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
14741            std::task::Poll::Ready(())
14742        })
14743        .await;
14744        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
14745        drop(owner);
14746        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
14747            .await
14748            .unwrap()
14749            .unwrap();
14750        assert_eq!(execution.runner.state.passes.len(), passes + 1);
14751        assert!(execution.runner.state.resource_block.is_none());
14752        assert_eq!(
14753            execution.runner.state.liveness(false),
14754            crate::run::Liveness::Live
14755        );
14756        drop(execution);
14757        assert!(RunState::load(&id).unwrap().driver_exited);
14758    }
14759
14760    #[tokio::test]
14761    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14762        crate::run::pin_test_home();
14763        let mut runner = runner_at(RunStatus::Gating);
14764        runner.state.save().unwrap();
14765        // run.json can commit, but the subsequent lease metadata write fails.
14766        // Cleanup must run after releasing the short ownership lock.
14767        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14768        assert!(runner.claim_driver().await.is_err());
14769        assert!(runner.state.driver_exited);
14770        assert_eq!(runner.state.driver_pid, None);
14771        let disk = RunState::load(&runner.state.id).unwrap();
14772        assert!(disk.driver_exited);
14773        assert_eq!(disk.driver_pid, None);
14774        assert_eq!(disk.status, RunStatus::Gating);
14775    }
14776
14777    #[tokio::test]
14778    async fn an_early_graph_error_releases_its_driver() {
14779        crate::run::pin_test_home();
14780        let mut runner = runner_at(RunStatus::Blocked);
14781        runner.state.config.graph.selected_implementer = Some("selected".into());
14782        runner.state.save().unwrap();
14783        assert!(runner.execute().await.is_err());
14784        let disk = RunState::load(&runner.state.id).unwrap();
14785        assert!(disk.driver_exited);
14786        assert_eq!(disk.driver_pid, None);
14787        assert_eq!(disk.status, RunStatus::Blocked);
14788    }
14789
14790    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14791        crate::run::MergeRepair {
14792            question: "repair-question".into(),
14793            head: "old-head".into(),
14794            pr: "https://github.com/o/r/pull/42".into(),
14795            reviews: vec![review_round(false, 1, 1, 1, false, true)],
14796            gate: vec![outcome(Some(0), false)],
14797            gate_head: Some("old-head".into()),
14798            gate_fixes: Vec::new(),
14799            pre_gate: Vec::new(),
14800            phase,
14801            failure: None,
14802        }
14803    }
14804
14805    #[tokio::test]
14806    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14807        crate::run::pin_test_home();
14808        let mut runner = runner_at(RunStatus::Landing);
14809        runner.state.merge_repairs.push(merge_repair_request(
14810            crate::run::MergeRepairPhase::Requested,
14811        ));
14812        runner.state.save().unwrap();
14813        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14814        assert!(
14815            runner
14816                .execute()
14817                .await
14818                .unwrap_err()
14819                .to_string()
14820                .contains("already running")
14821        );
14822        assert_eq!(
14823            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14824            crate::run::MergeRepairPhase::Requested
14825        );
14826        drop(claim);
14827        let mut disk = runner.state.clone();
14828        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14829        disk.save().unwrap();
14830        assert!(
14831            runner
14832                .execute()
14833                .await
14834                .unwrap_err()
14835                .to_string()
14836                .contains("decision changed")
14837        );
14838        assert_eq!(
14839            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14840            crate::run::MergeRepairPhase::Complete
14841        );
14842    }
14843
14844    #[test]
14845    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14846        crate::run::pin_test_home();
14847        let mut runner = runner_at(RunStatus::Landing);
14848        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14849        runner.state.reviews = request.reviews.clone();
14850        runner.state.gate = request.gate.clone();
14851        runner.state.gate_ran = true;
14852        runner.state.land_approval = Some(crate::run::LandApproval {
14853            question: "old".into(),
14854            head: "old-head".into(),
14855        });
14856        runner.state.merge_repairs.push(request);
14857        runner.begin_merge_repair_review(0).unwrap();
14858        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14859        assert!(!runner.state.gate_status().ok());
14860        assert!(runner.state.land_approval.is_none());
14861        let disk = RunState::load(&runner.state.id).unwrap();
14862        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14863        assert!(disk.merge_repairs[0].gate[0].ok());
14864        assert_eq!(
14865            disk.merge_repairs[0].phase,
14866            crate::run::MergeRepairPhase::Reviewing
14867        );
14868        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14869        assert!(disk.reviews.is_empty());
14870    }
14871
14872    #[tokio::test]
14873    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14874        crate::run::pin_test_home();
14875        for phase in [
14876            crate::run::MergeRepairPhase::Fixing,
14877            crate::run::MergeRepairPhase::Requested,
14878        ] {
14879            let mut runner = runner_at(RunStatus::Landing);
14880            runner.state.config.graph.land_rounds = 0;
14881            runner.state.merge_repairs.push(merge_repair_request(phase));
14882            runner.state.save().unwrap();
14883            assert!(!runner.repair_merge_review().await.unwrap());
14884            assert_eq!(runner.state.status, RunStatus::Blocked);
14885            assert_eq!(
14886                runner.state.merge_repairs[0].phase,
14887                crate::run::MergeRepairPhase::Failed
14888            );
14889            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14890            assert!(
14891                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14892                    "interrupted"
14893                } else {
14894                    "budget exhausted"
14895                })
14896            );
14897            assert!(runner.state.merge.is_none());
14898        }
14899    }
14900
14901    /// `park_here` folding in the reason `Pause::park_because` recorded -
14902    /// this is what lets an operator reading a run's events tell an
14903    /// interrupt-driven park from an ordinary shutdown park.
14904    #[test]
14905    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14906        crate::run::pin_test_home();
14907        let mut runner = runner_at(RunStatus::Implementing);
14908        let interrupt = Pause::new();
14909        runner.watch_interrupt(interrupt.clone());
14910
14911        interrupt.park_because("task a1b2 asked to run first");
14912
14913        assert!(runner.park_here().expect("park_here"));
14914        assert!(runner.state.parked);
14915        let last = runner.state.events.last().expect("a park event");
14916        assert_eq!(last.node, "park");
14917        assert!(
14918            last.message.contains("task a1b2 asked to run first"),
14919            "expected the interrupt reason in {:?}",
14920            last.message
14921        );
14922    }
14923
14924    /// The cause is stored on the pass when it parks, and the first cause a
14925    /// shared pause was given is the one that stays.
14926    #[test]
14927    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14928        crate::run::pin_test_home();
14929        let mut runner = runner_at(RunStatus::Implementing);
14930        runner.state.begin_pass();
14931        let shutdown = Pause::new();
14932        runner.on_pause(shutdown.clone());
14933        let upgrade = ParkCause::Upgrade {
14934            from: "0.121.0".to_owned(),
14935            to: Some("0.121.1".to_owned()),
14936        };
14937        shutdown.park_for(upgrade.clone());
14938        shutdown.park_for(ParkCause::Operator);
14939
14940        assert!(runner.park_here().expect("park_here"));
14941        assert_eq!(
14942            runner.state.passes[0].end,
14943            Some(crate::run::PassEnd::Parked { cause: upgrade })
14944        );
14945        let last = runner.state.events.last().expect("a park event");
14946        assert!(last.message.contains("0.121.0"), "{}", last.message);
14947    }
14948
14949    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
14950    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
14951    /// cleared) must not make a *different* run - one only watching its own,
14952    /// unshared interrupt `Pause` - see itself as parked. If a future change
14953    /// ever collapsed these back into one handle, the interrupt scheduler
14954    /// would park every run for the rest of the daemon's life, not just the
14955    /// one it meant to interrupt.
14956    #[test]
14957    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14958        crate::run::pin_test_home();
14959        let mut runner = runner_at(RunStatus::Implementing);
14960        let shutdown = Pause::new();
14961        runner.on_pause(shutdown.clone());
14962        let interrupt = Pause::new();
14963        runner.watch_interrupt(interrupt.clone());
14964
14965        // Nobody has asked for anything yet.
14966        assert!(!runner.park_here().expect("park_here"));
14967        assert!(!runner.state.parked);
14968
14969        // Only the interrupt handle fires; the shutdown handle stays clear.
14970        interrupt.park_because("test");
14971        assert!(!shutdown.parked());
14972        assert!(runner.park_here().expect("park_here"));
14973    }
14974
14975    /// The property every prior attempt at this feature failed to pin down:
14976    /// asking a run to park while one of its nodes has a real, in-flight
14977    /// async operation running (an agent call, in production) must not cut
14978    /// that operation short. `park_here` is only ever consulted *between*
14979    /// `execute`'s node calls - see its own doc - so nothing inside a node
14980    /// can observe a park request until the node itself returns. This proves
14981    /// that structurally, with real `tokio` concurrency and a channel
14982    /// handshake (never a sleep, which would only prove "usually", not
14983    /// "cannot"): the "node" below reports that it has genuinely started,
14984    /// and only then is the park requested; the node still has to be told to
14985    /// finish before `park_here` is ever called, exactly mirroring every
14986    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
14987    /// in `execute`.
14988    #[tokio::test]
14989    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14990        crate::run::pin_test_home();
14991        let mut runner = runner_at(RunStatus::Implementing);
14992        let interrupt = Pause::new();
14993        runner.watch_interrupt(interrupt.clone());
14994
14995        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14996        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14997
14998        // Stands in for one node's in-flight agent call: it proves it has
14999        // genuinely started, then blocks - exactly as a spawned CLI process
15000        // does - until told to finish.
15001        let node = async move {
15002            started_tx.send(()).expect("send started");
15003            finish_rx.await.expect("recv finish");
15004            "node finished"
15005        };
15006
15007        let interrupter = async move {
15008            started_rx.await.expect("recv started");
15009            // The call is now genuinely in flight. Ask it to park.
15010            interrupt.park_because("higher-priority task waiting");
15011            // Nothing the node does can observe this yet - there is no
15012            // check inside it, by construction - so let the executor run
15013            // anything pending and then let the node finish on its own.
15014            tokio::task::yield_now().await;
15015            finish_tx.send(()).expect("send finish");
15016        };
15017
15018        let (node_result, ()) = tokio::join!(node, interrupter);
15019        assert_eq!(
15020            node_result, "node finished",
15021            "the in-flight call ran to completion"
15022        );
15023
15024        // Only now, at the boundary the real `execute` would check right
15025        // after this node, does the park take effect.
15026        assert!(runner.park_here().expect("park_here"));
15027        assert!(runner.state.parked);
15028    }
15029
15030    /// A run parked mid-competition carries every field it had accumulated
15031    /// through the exact same disk round-trip an ordinary resume uses -
15032    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
15033    /// Nothing about parking for an interrupt is a special case of that path;
15034    /// this is what proves it rather than assuming it.
15035    #[test]
15036    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
15037        crate::run::pin_test_home();
15038        let mut runner = runner_at(RunStatus::Judging);
15039        // `Runner::resume` re-resolves roles from the saved config, which
15040        // refuses an empty roster - give it the same minimal one `conductor`
15041        // itself uses.
15042        runner.state.config.agents = vec![conductor()];
15043        runner.state.candidates = vec![Candidate {
15044            index: 0,
15045            label: 'A',
15046            agent: "alpha".to_owned(),
15047            branch: "magi/x/A".to_owned(),
15048            worktree: PathBuf::from("/nonexistent/worktree"),
15049            summary: "did the thing".to_owned(),
15050            stat: "1 file changed".to_owned(),
15051            files: 1,
15052            commits: 1,
15053            empty: false,
15054            failed: None,
15055            verified_noop: None,
15056            duration_ms: 1234,
15057            folded: false,
15058        }];
15059        let run_id = runner.state.id.clone();
15060
15061        let interrupt = Pause::new();
15062        runner.watch_interrupt(interrupt.clone());
15063        interrupt.park_because("task c3d4 asked to run first");
15064        assert!(runner.park_here().expect("park_here"));
15065
15066        let resumed = Runner::resume(&run_id).expect("resume");
15067        assert_eq!(resumed.state.candidates.len(), 1);
15068        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
15069        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
15070        assert_eq!(resumed.state.status, runner.state.status);
15071        assert!(
15072            resumed.state.parked,
15073            "still parked until `execute` actually walks the graph again"
15074        );
15075        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
15076    }
15077
15078    /// A fresh open question on `run`, stored and handed back for assertions.
15079    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
15080        let mut q = ask::Question::new(
15081            run.to_owned(),
15082            "implement".to_owned(),
15083            "impl-A".to_owned(),
15084            "Which storage backend should the cache use?".to_owned(),
15085            String::new(),
15086            vec!["SQLite".to_owned(), "Redis".to_owned()],
15087        );
15088        store.put(&mut q).unwrap();
15089        q
15090    }
15091
15092    #[test]
15093    fn a_failed_runs_open_question_is_abandoned() {
15094        ask_test_home();
15095        let store = ask::Questions::open();
15096        let mut runner = runner_at(RunStatus::Failed);
15097        let run = runner.state.id.clone();
15098        let q = ask_open_question(&store, &run);
15099
15100        runner.settle_questions();
15101
15102        let back = store.get(&q.id).unwrap();
15103        assert!(
15104            !back.status.open(),
15105            "the seat that asked died with the run; nobody is left to read an answer"
15106        );
15107        assert!(
15108            back.detail.contains(&run) && back.detail.contains("failed"),
15109            "the reason names what the run became, not just that it is gone: {}",
15110            back.detail
15111        );
15112    }
15113
15114    #[test]
15115    fn a_merged_runs_open_question_is_abandoned_too() {
15116        ask_test_home();
15117        let store = ask::Questions::open();
15118        // A run that finishes cleanly still leaves nobody to read an answer -
15119        // this is not only a failure-path cleanup.
15120        for status in [RunStatus::Merged, RunStatus::Ready] {
15121            let mut runner = runner_at(status);
15122            let run = runner.state.id.clone();
15123            let q = ask_open_question(&store, &run);
15124
15125            runner.settle_questions();
15126
15127            let back = store.get(&q.id).unwrap();
15128            assert!(
15129                !back.status.open(),
15130                "{status:?} run's question must not outlive the run"
15131            );
15132        }
15133    }
15134
15135    #[test]
15136    fn a_still_resumable_runs_open_question_is_left_alone() {
15137        ask_test_home();
15138        let store = ask::Questions::open();
15139        // `Blocked` and `Stalled` can still be resumed — the candidates, the
15140        // review round and the seat sessions are all still on disk — so a
15141        // question asked mid-round may yet get a real answer from a real
15142        // resume. Sweeping it here would be exactly the failure mode this
15143        // whole feature exists to avoid on the other side.
15144        for status in [RunStatus::Blocked, RunStatus::Stalled] {
15145            let mut runner = runner_at(status);
15146            let run = runner.state.id.clone();
15147            let q = ask_open_question(&store, &run);
15148
15149            runner.settle_questions();
15150
15151            let back = store.get(&q.id).unwrap();
15152            assert!(
15153                back.status.open(),
15154                "{status:?} is still alive; the question must still be waiting"
15155            );
15156        }
15157    }
15158
15159    #[test]
15160    fn settle_questions_never_touches_an_already_answered_question() {
15161        ask_test_home();
15162        let store = ask::Questions::open();
15163        let mut runner = runner_at(RunStatus::Failed);
15164        let run = runner.state.id.clone();
15165        let mut q = ask_open_question(&store, &run);
15166        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15167            .unwrap();
15168        store.put(&mut q).unwrap();
15169
15170        // Called twice, the way a crash-recovered daemon reclaim and the
15171        // graph's own cleanup both can for the same run — `abandon_for_run`
15172        // only ever touches what is still open, so this must be inert both
15173        // times, not merely the second.
15174        runner.settle_questions();
15175        runner.settle_questions();
15176
15177        let back = store.get(&q.id).unwrap();
15178        assert_eq!(
15179            back.status,
15180            ask::QuestionStatus::Answered,
15181            "a real answer is a decision on record, never overwritten by a sweep"
15182        );
15183    }
15184
15185    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
15186    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
15187    /// without merging, whose winner is still the operator's answer to read.
15188    /// Nothing previously called `fold_run` itself with a real `tally`, so
15189    /// this is the first test to pin down the one distinction the whole
15190    /// automatic-fold feature depends on: the winner's worktree and branch
15191    /// must survive, everything else sharing the run's worktree bay - a
15192    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
15193    /// second sweep treats every non-winner directory under the bay alike -
15194    /// must not.
15195    #[tokio::test]
15196    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15197        crate::run::pin_test_home();
15198        let tmp = tempfile::tempdir().expect("tempdir");
15199        let repo = tmp.path().join("repo");
15200        std::fs::create_dir_all(&repo).unwrap();
15201        init_repo(&repo);
15202
15203        let mut config = Config::default();
15204        config.graph.worktree_root = Some(tmp.path().join("wt"));
15205
15206        let mut state = RunState::new(
15207            repo.clone(),
15208            "main".to_owned(),
15209            "deadbeef".to_owned(),
15210            "task".to_owned(),
15211            config,
15212        );
15213        let root = state.worktree_root();
15214        let wt_a = root.join("cand-A");
15215        let wt_b = root.join("cand-B");
15216        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15217            .await
15218            .expect("worktree A");
15219        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15220            .await
15221            .expect("worktree B");
15222
15223        state.candidates = vec![
15224            Candidate {
15225                index: 0,
15226                label: 'A',
15227                agent: "alpha".to_owned(),
15228                branch: "magi/x/A".to_owned(),
15229                worktree: wt_a.clone(),
15230                summary: String::new(),
15231                stat: String::new(),
15232                files: 0,
15233                commits: 0,
15234                empty: false,
15235                failed: None,
15236                verified_noop: None,
15237                duration_ms: 0,
15238                folded: false,
15239            },
15240            Candidate {
15241                index: 1,
15242                label: 'B',
15243                agent: "beta".to_owned(),
15244                branch: "magi/x/B".to_owned(),
15245                worktree: wt_b.clone(),
15246                summary: String::new(),
15247                stat: String::new(),
15248                files: 0,
15249                commits: 0,
15250                empty: false,
15251                failed: None,
15252                verified_noop: None,
15253                duration_ms: 0,
15254                folded: false,
15255            },
15256        ];
15257        state.tally = Some(Tally {
15258            first_choice: BTreeMap::from([('A', 1)]),
15259            borda: BTreeMap::new(),
15260            winner: 'A',
15261            rankings: 1,
15262            unanimous_initial: true,
15263            deliberated: false,
15264            changed_votes: 0,
15265            unanimous_final: true,
15266            tie_break: None,
15267            judges: 1,
15268            present: 1,
15269            quorum: 1,
15270            met_quorum: true,
15271            uncontested: None,
15272        });
15273        state.status = RunStatus::Ready;
15274
15275        fold_run(&mut state, false, &crate::run::home())
15276            .await
15277            .expect("fold_run");
15278
15279        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15280        assert!(
15281            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15282            "the unmerged winner's branch survives"
15283        );
15284        assert!(
15285            !state.candidates[0].folded,
15286            "the winner is not marked folded"
15287        );
15288
15289        assert!(!wt_b.exists(), "the loser's worktree is removed");
15290        assert!(
15291            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15292            "the loser's branch is removed"
15293        );
15294        assert!(state.candidates[1].folded, "the loser is marked folded");
15295    }
15296
15297    #[tokio::test]
15298    #[cfg(unix)]
15299    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15300        use std::os::fd::AsRawFd as _;
15301        use std::time::Duration;
15302
15303        let tmp = tempfile::tempdir().unwrap();
15304        let repo = tmp.path().join("repo");
15305        std::fs::create_dir_all(&repo).unwrap();
15306        init_repo(&repo);
15307        let common = repo.join(".git");
15308        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15309        let home = tmp.path().join("home");
15310        let mut config = Config::default();
15311        config.coordination = crate::config::test_coordination();
15312        config.graph.worktree_root = Some(tmp.path().join("wt"));
15313        config.graph.timeout_resource_wait = 1;
15314        let mut state = RunState::new(
15315            repo.clone(),
15316            "main".into(),
15317            "deadbeef".into(),
15318            "task".into(),
15319            config,
15320        );
15321        let root = state.worktree_root();
15322        let candidate = root.join("cand-A");
15323        let reviewer = root.join("review-1");
15324        let branch = state.branch_for('A');
15325        git::worktree_add_branch(&repo, &candidate, &branch, "main")
15326            .await
15327            .unwrap();
15328        git::worktree_add_detached(&repo, &reviewer, "main")
15329            .await
15330            .unwrap();
15331        git::acquire_worktree_config(&repo).await.unwrap();
15332        state.enabled_worktree_config = true;
15333        state.status = RunStatus::Ready;
15334        state.candidates.push(Candidate {
15335            index: 0,
15336            label: 'A',
15337            agent: "implement".into(),
15338            branch: branch.clone(),
15339            worktree: candidate.clone(),
15340            summary: String::new(),
15341            stat: String::new(),
15342            files: 0,
15343            commits: 0,
15344            empty: false,
15345            failed: None,
15346            verified_noop: None,
15347            duration_ms: 0,
15348            folded: false,
15349        });
15350        state.save_under(&home).unwrap();
15351        let config_before = std::fs::read(common.join("config")).unwrap();
15352        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15353            .await
15354            .unwrap();
15355        // A separate OS handle bypasses the launcher's local gate, exactly as
15356        // an external updater does, and must still prevent all folding work.
15357        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15358        assert_eq!(
15359            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15360            0
15361        );
15362        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15363        assert!(
15364            tokio::time::timeout(Duration::from_millis(200), &mut fold)
15365                .await
15366                .is_err()
15367        );
15368        assert!(candidate.exists());
15369        assert!(reviewer.exists());
15370        assert!(git::branch_exists(&repo, &branch).await.unwrap());
15371        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15372        assert_eq!(
15373            git::git(&repo, &["worktree", "list", "--porcelain"])
15374                .await
15375                .unwrap(),
15376            worktrees_before
15377        );
15378        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15379            .await
15380            .unwrap()
15381            .unwrap_err();
15382        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15383        drop(fold);
15384        assert!(candidate.exists() && reviewer.exists());
15385        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15386        assert_eq!(
15387            state.resource_block.as_ref().unwrap().kind,
15388            "repository coordination"
15389        );
15390        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15391        drop(external);
15392        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15393        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15394            .await
15395            .unwrap()
15396            .unwrap();
15397        drop(fold);
15398        assert!(removed.contains(&branch));
15399        assert!(!root.exists());
15400        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15401        assert!(state.candidates[0].folded);
15402        assert!(!state.enabled_worktree_config);
15403        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15404        let enabled = git::git_raw(
15405            &repo,
15406            &["config", "--local", "--get", "extensions.worktreeConfig"],
15407        )
15408        .await
15409        .unwrap();
15410        assert!(!enabled.ok(), "the last config reference was released");
15411    }
15412
15413    /// A branch handed to a later run is that run's (and its pull request's):
15414    /// folding the run that released it must not delete it.
15415    #[tokio::test]
15416    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15417        let tmp = tempfile::tempdir().expect("tempdir");
15418        let repo = tmp.path().join("repo");
15419        std::fs::create_dir_all(&repo).unwrap();
15420        init_repo(&repo);
15421        let home = tmp.path().join("home");
15422
15423        let mut config = Config::default();
15424        config.graph.worktree_root = Some(tmp.path().join("wt"));
15425        let mut state = RunState::new(
15426            repo.clone(),
15427            "main".to_owned(),
15428            "deadbeef".to_owned(),
15429            "task".to_owned(),
15430            config,
15431        );
15432        // The worktree is already gone (released); the branch survives.
15433        git::git(&repo, &["branch", "magi/x/A", "main"])
15434            .await
15435            .expect("branch");
15436        state.candidates = vec![Candidate {
15437            index: 0,
15438            label: 'A',
15439            agent: "alpha".to_owned(),
15440            branch: "magi/x/A".to_owned(),
15441            worktree: state.worktree_root().join("cand-A"),
15442            summary: String::new(),
15443            stat: String::new(),
15444            files: 0,
15445            commits: 0,
15446            empty: false,
15447            failed: None,
15448            verified_noop: None,
15449            duration_ms: 0,
15450            folded: true,
15451        }];
15452        state.released_to = Some("20260901-000000-new1".to_owned());
15453        state.released_branches = vec!["magi/x/A".to_owned()];
15454
15455        fold_run(&mut state, true, &home).await.expect("fold_run");
15456        state.status = RunStatus::Merged;
15457        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15458        state.merge = Some(crate::run::MergeOutcome {
15459            mode: MergeMode::Pr,
15460            ok: true,
15461            detail: "confirmed merged".into(),
15462            empty: false,
15463        });
15464        fold_run(&mut state, true, &home)
15465            .await
15466            .expect("merged fold");
15467
15468        assert!(
15469            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15470            "the handed-over branch survives a fold"
15471        );
15472    }
15473
15474    /// A winner with nothing ahead of the base is caught before `gh` is ever
15475    /// asked for a pull request, and the message carries what the task's
15476    /// references resolved to.
15477    #[tokio::test]
15478    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15479        let tmp = tempfile::tempdir().expect("tempdir");
15480        let repo = tmp.path().join("repo");
15481        std::fs::create_dir_all(&repo).unwrap();
15482        init_repo(&repo);
15483        let run = |args: &[&str]| {
15484            let out = std::process::Command::new("git")
15485                .quiet()
15486                .args(args)
15487                .current_dir(&repo)
15488                .output()
15489                .expect("spawn git");
15490            assert!(out.status.success(), "git {args:?}");
15491        };
15492        run(&["branch", "magi/x/A"]);
15493        run(&["checkout", "-q", "-b", "magi/x/B"]);
15494        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15495        run(&["add", "-A"]);
15496        run(&["commit", "-q", "-m", "work"]);
15497        run(&["checkout", "-q", "main"]);
15498
15499        // A pull request is compared against the remote's base, so the
15500        // fixture needs one. Before it exists nothing can be read, and the
15501        // answer must be "not empty".
15502        let probe = RunState::new(
15503            repo.clone(),
15504            "main".to_owned(),
15505            "deadbeef".to_owned(),
15506            "task".to_owned(),
15507            Config::default(),
15508        );
15509        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15510        let bare = tmp.path().join("origin.git");
15511        let out = std::process::Command::new("git")
15512            .quiet()
15513            .args(["init", "-q", "--bare"])
15514            .arg(&bare)
15515            .output()
15516            .expect("spawn git");
15517        assert!(out.status.success(), "git init --bare");
15518        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15519        run(&["push", "-q", "origin", "main"]);
15520
15521        let mut state = RunState::new(
15522            repo.clone(),
15523            "main".to_owned(),
15524            "deadbeef".to_owned(),
15525            "task".to_owned(),
15526            Config::default(),
15527        );
15528        state.seeds = vec![refs::Seed {
15529            token: "magi/27b2/A".to_owned(),
15530            kind: refs::SeedKind::Unresolved,
15531            sha: String::new(),
15532            branch: true,
15533            detail: "no branch or commit named magi/27b2/A".to_owned(),
15534        }];
15535
15536        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15537        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15538        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15539        let detail = empty_candidate_detail(&state, "main");
15540        assert!(detail.starts_with("empty candidate"), "{detail}");
15541        assert!(!detail.contains("magi/27b2/A"), "{detail}");
15542        assert!(detail.contains("--start-from"), "{detail}");
15543    }
15544
15545    /// `status == Ready` used to be read as "this is the harmless
15546    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
15547    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
15548    /// run's PR was closed without merging — and reentering `merge` with
15549    /// `mode` still `Pr` does not know the difference, so it pushed and
15550    /// opened a second pull request. `mode == Local` reproduces the same
15551    /// blind spot without a network call: reentry must not attempt another
15552    /// git merge once this node has already recorded an outcome.
15553    #[tokio::test]
15554    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15555        let tmp = tempfile::tempdir().expect("tempdir");
15556        let repo = tmp.path().join("repo");
15557        std::fs::create_dir_all(&repo).unwrap();
15558        init_repo(&repo);
15559
15560        let mut config = Config::default();
15561        config.merge.mode = MergeMode::Local;
15562
15563        let mut state = RunState::new(
15564            repo.clone(),
15565            "main".to_owned(),
15566            "deadbeef".to_owned(),
15567            "task".to_owned(),
15568            config,
15569        );
15570        state.candidates = vec![Candidate {
15571            index: 0,
15572            label: 'A',
15573            agent: "alpha".to_owned(),
15574            branch: "does-not-exist".to_owned(),
15575            worktree: repo.clone(),
15576            summary: String::new(),
15577            stat: String::new(),
15578            files: 0,
15579            commits: 0,
15580            empty: false,
15581            failed: None,
15582            verified_noop: None,
15583            duration_ms: 0,
15584            folded: false,
15585        }];
15586        state.tally = Some(Tally {
15587            first_choice: BTreeMap::from([('A', 1)]),
15588            borda: BTreeMap::new(),
15589            winner: 'A',
15590            rankings: 1,
15591            unanimous_initial: true,
15592            deliberated: false,
15593            changed_votes: 0,
15594            unanimous_final: true,
15595            tie_break: None,
15596            judges: 0,
15597            present: 0,
15598            quorum: 0,
15599            met_quorum: true,
15600            uncontested: Some("only candidate A produced a change".to_owned()),
15601        });
15602        state.reviews = vec![ReviewRound {
15603            round: 1,
15604            head: "deadbeef".to_owned(),
15605            verified_head: None,
15606            verified_at: None,
15607            reviews: Vec::new(),
15608            e2e: Vec::new(),
15609            fix: None,
15610            blocking: 0,
15611            answered: 0,
15612            expected: 0,
15613            clean: true,
15614            verify_retried: false,
15615            e2e_deferred: false,
15616            e2e_defer_reason: None,
15617            progressed: false,
15618            vote_split: false,
15619            reconsideration: Vec::new(),
15620            verdict: None,
15621        }];
15622        state.gate = vec![CommandOutcome {
15623            command: "test".to_owned(),
15624            code: Some(0),
15625            output_tail: String::new(),
15626            duration_ms: 0,
15627            resource_blocked: false,
15628        }];
15629        state.gate_ran = true;
15630        // Reached its conclusion already — e.g. `land` closing the PR without
15631        // merging it, which (like the honest `MergeMode::None` path) leaves
15632        // `status` at `Ready`. The recorded outcome is what actually marks
15633        // this node done.
15634        state.status = RunStatus::Ready;
15635        state.merge = Some(MergeOutcome {
15636            mode: MergeMode::Local,
15637            ok: false,
15638            detail: "already concluded".to_owned(),
15639            empty: false,
15640        });
15641
15642        let mut runner = Runner {
15643            state,
15644            roles: ResolvedRoles {
15645                implementers: Vec::new(),
15646                judges: Vec::new(),
15647                reviewers: Vec::new(),
15648                fixer: None,
15649                conductor: conductor(),
15650                implementer_roster: Vec::new(),
15651                judge_roster: Vec::new(),
15652                reviewer_roster: Vec::new(),
15653            },
15654            sem: Arc::new(Semaphore::new(1)),
15655            pause: Pause::new(),
15656            interrupt: Pause::new(),
15657        };
15658
15659        runner.merge().await.expect("merge");
15660
15661        assert_eq!(
15662            runner.state.status,
15663            RunStatus::Ready,
15664            "a concluded run's status must not change on reentry"
15665        );
15666        assert_eq!(
15667            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15668            Some("already concluded"),
15669            "merge must not run again once the node already recorded an outcome"
15670        );
15671    }
15672
15673    /// `gate` leaves `state.gate_ran` false both before it has ever run and
15674    /// when its last attempt was resource-blocked (the shared build cache
15675    /// could not be acquired or confirmed fresh in time - see
15676    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
15677    /// `Vec` this also leaves behind used to read as "nothing failed" and let
15678    /// a run merge a tree the gate never actually checked - exactly the case
15679    /// a contended cache produces on every retry until it clears. `merge`
15680    /// must refuse until `gate` has actually recorded an attempt.
15681    #[tokio::test]
15682    async fn merge_refuses_a_gate_that_has_not_actually_run() {
15683        let tmp = tempfile::tempdir().expect("tempdir");
15684        let repo = tmp.path().join("repo");
15685        std::fs::create_dir_all(&repo).unwrap();
15686        init_repo(&repo);
15687
15688        let mut config = Config::default();
15689        config.merge.mode = MergeMode::Local;
15690
15691        let mut state = RunState::new(
15692            repo.clone(),
15693            "main".to_owned(),
15694            "deadbeef".to_owned(),
15695            "task".to_owned(),
15696            config,
15697        );
15698        state.candidates = vec![Candidate {
15699            index: 0,
15700            label: 'A',
15701            agent: "alpha".to_owned(),
15702            branch: "does-not-exist".to_owned(),
15703            worktree: repo.clone(),
15704            summary: String::new(),
15705            stat: String::new(),
15706            files: 0,
15707            commits: 0,
15708            empty: false,
15709            failed: None,
15710            verified_noop: None,
15711            duration_ms: 0,
15712            folded: false,
15713        }];
15714        state.tally = Some(Tally {
15715            first_choice: BTreeMap::from([('A', 1)]),
15716            borda: BTreeMap::new(),
15717            winner: 'A',
15718            rankings: 1,
15719            unanimous_initial: true,
15720            deliberated: false,
15721            changed_votes: 0,
15722            unanimous_final: true,
15723            tie_break: None,
15724            judges: 0,
15725            present: 0,
15726            quorum: 0,
15727            met_quorum: true,
15728            uncontested: Some("only candidate A produced a change".to_owned()),
15729        });
15730        state.reviews = vec![ReviewRound {
15731            round: 1,
15732            head: "deadbeef".to_owned(),
15733            verified_head: None,
15734            verified_at: None,
15735            reviews: Vec::new(),
15736            e2e: Vec::new(),
15737            fix: None,
15738            blocking: 0,
15739            answered: 0,
15740            expected: 0,
15741            clean: true,
15742            verify_retried: false,
15743            e2e_deferred: false,
15744            e2e_defer_reason: None,
15745            progressed: false,
15746            vote_split: false,
15747            reconsideration: Vec::new(),
15748            verdict: None,
15749        }];
15750        // The point: `gate` has not recorded anything yet.
15751        state.gate = Vec::new();
15752        state.gate_ran = false;
15753        state.status = RunStatus::Gating;
15754
15755        let mut runner = Runner {
15756            state,
15757            roles: ResolvedRoles {
15758                implementers: Vec::new(),
15759                judges: Vec::new(),
15760                reviewers: Vec::new(),
15761                fixer: None,
15762                conductor: conductor(),
15763                implementer_roster: Vec::new(),
15764                judge_roster: Vec::new(),
15765                reviewer_roster: Vec::new(),
15766            },
15767            sem: Arc::new(Semaphore::new(1)),
15768            pause: Pause::new(),
15769            interrupt: Pause::new(),
15770        };
15771
15772        runner.merge().await.expect("merge");
15773
15774        assert!(
15775            runner.state.merge.is_none(),
15776            "an empty gate must never be read as a passing one: {:?}",
15777            runner.state.merge
15778        );
15779    }
15780
15781    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
15782    /// commands configured and `merge.mode` is `none` (a review-only run).
15783    /// `gate` must still record a real attempt — zero commands, vacuously
15784    /// passed — rather than leaving `state.gate` empty in a way `merge`
15785    /// cannot tell apart from "never ran"; otherwise the run reaches
15786    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
15787    #[tokio::test]
15788    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15789        let tmp = tempfile::tempdir().expect("tempdir");
15790        let repo = tmp.path().join("repo");
15791        std::fs::create_dir_all(&repo).unwrap();
15792        init_repo(&repo);
15793
15794        // Default config: `verify.gate` empty, `merge.mode` is `none`.
15795        let config = Config::default();
15796
15797        let mut state = RunState::new(
15798            repo.clone(),
15799            "main".to_owned(),
15800            "deadbeef".to_owned(),
15801            "task".to_owned(),
15802            config,
15803        );
15804        state.candidates = vec![Candidate {
15805            index: 0,
15806            label: 'A',
15807            agent: "alpha".to_owned(),
15808            branch: "does-not-exist".to_owned(),
15809            worktree: repo.clone(),
15810            summary: String::new(),
15811            stat: String::new(),
15812            files: 0,
15813            commits: 0,
15814            empty: false,
15815            failed: None,
15816            verified_noop: None,
15817            duration_ms: 0,
15818            folded: false,
15819        }];
15820        state.tally = Some(Tally {
15821            first_choice: BTreeMap::from([('A', 1)]),
15822            borda: BTreeMap::new(),
15823            winner: 'A',
15824            rankings: 1,
15825            unanimous_initial: true,
15826            deliberated: false,
15827            changed_votes: 0,
15828            unanimous_final: true,
15829            tie_break: None,
15830            judges: 0,
15831            present: 0,
15832            quorum: 0,
15833            met_quorum: true,
15834            uncontested: Some("only candidate A produced a change".to_owned()),
15835        });
15836        state.reviews = vec![ReviewRound {
15837            round: 1,
15838            head: "deadbeef".to_owned(),
15839            verified_head: None,
15840            verified_at: None,
15841            reviews: Vec::new(),
15842            e2e: Vec::new(),
15843            fix: None,
15844            blocking: 0,
15845            answered: 0,
15846            expected: 0,
15847            clean: true,
15848            verify_retried: false,
15849            e2e_deferred: false,
15850            e2e_defer_reason: None,
15851            progressed: false,
15852            vote_split: false,
15853            reconsideration: Vec::new(),
15854            verdict: None,
15855        }];
15856
15857        let mut runner = Runner {
15858            state,
15859            roles: ResolvedRoles {
15860                implementers: Vec::new(),
15861                judges: Vec::new(),
15862                reviewers: Vec::new(),
15863                fixer: None,
15864                conductor: conductor(),
15865                implementer_roster: Vec::new(),
15866                judge_roster: Vec::new(),
15867                reviewer_roster: Vec::new(),
15868            },
15869            sem: Arc::new(Semaphore::new(1)),
15870            pause: Pause::new(),
15871            interrupt: Pause::new(),
15872        };
15873
15874        runner.gate().await.expect("gate");
15875        assert!(
15876            runner.state.gate_ran,
15877            "zero configured commands is still a real attempt, not an unrun gate"
15878        );
15879        assert!(runner.state.gate.is_empty());
15880        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15881        assert_ne!(
15882            runner.state.status,
15883            RunStatus::Blocked,
15884            "a gate with nothing to check must not read as failed"
15885        );
15886
15887        runner.merge().await.expect("merge");
15888        assert_eq!(
15889            runner.state.status,
15890            RunStatus::Ready,
15891            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15892        );
15893    }
15894
15895    /// `Config::cache_dir` is derived from `verify.e2e` as well as
15896    /// `verify.gate` (so the e2e leg and the final gate never build against
15897    /// different directories). With zero `verify.gate` commands but a
15898    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
15899    /// that lease before discovering it had nothing to run - so a repo with
15900    /// no gate commands could come back `resource_blocked` (and therefore
15901    /// still `gate_ran == false`) on nothing but an unrelated run holding the
15902    /// cache, exactly the contention this run's own zero commands could
15903    /// never have touched. `gate` must recognise there is nothing to check
15904    /// before it ever asks for the lease.
15905    #[tokio::test]
15906    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15907        crate::run::pin_test_home();
15908        let home = crate::run::home();
15909
15910        let tmp = tempfile::tempdir().expect("tempdir");
15911        let repo = tmp.path().join("repo");
15912        std::fs::create_dir_all(&repo).unwrap();
15913        init_repo(&repo);
15914        // Unique to this test, so holding its lease cannot collide with
15915        // another test sharing the same process-wide `home`.
15916        let cache_dir = tmp.path().join("target");
15917
15918        let mut config = Config::default();
15919        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15920        // `verify.gate` stays empty (the default). Bounded so a regression
15921        // that does start waiting fails the test in seconds, not hangs it.
15922        config.graph.timeout_verify = Some(2);
15923        config.graph.timeout_resource_wait = 2;
15924
15925        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15926        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15927            .expect("no io error acquiring directly")
15928        {
15929            crate::cache::AcquireOutcome::Acquired(g) => g,
15930            crate::cache::AcquireOutcome::Busy(b) => {
15931                panic!("expected the direct acquire to win the lease first: {b:?}")
15932            }
15933        };
15934
15935        let mut state = RunState::new(
15936            repo.clone(),
15937            "main".to_owned(),
15938            "deadbeef".to_owned(),
15939            "task".to_owned(),
15940            config,
15941        );
15942        state.candidates = vec![Candidate {
15943            index: 0,
15944            label: 'A',
15945            agent: "alpha".to_owned(),
15946            branch: "does-not-exist".to_owned(),
15947            worktree: repo.clone(),
15948            summary: String::new(),
15949            stat: String::new(),
15950            files: 0,
15951            commits: 0,
15952            empty: false,
15953            failed: None,
15954            verified_noop: None,
15955            duration_ms: 0,
15956            folded: false,
15957        }];
15958        state.tally = Some(Tally {
15959            first_choice: BTreeMap::from([('A', 1)]),
15960            borda: BTreeMap::new(),
15961            winner: 'A',
15962            rankings: 1,
15963            unanimous_initial: true,
15964            deliberated: false,
15965            changed_votes: 0,
15966            unanimous_final: true,
15967            tie_break: None,
15968            judges: 0,
15969            present: 0,
15970            quorum: 0,
15971            met_quorum: true,
15972            uncontested: Some("only candidate A produced a change".to_owned()),
15973        });
15974        state.reviews = vec![ReviewRound {
15975            round: 1,
15976            head: "deadbeef".to_owned(),
15977            verified_head: None,
15978            verified_at: None,
15979            reviews: Vec::new(),
15980            e2e: Vec::new(),
15981            fix: None,
15982            blocking: 0,
15983            answered: 0,
15984            expected: 0,
15985            clean: true,
15986            verify_retried: false,
15987            e2e_deferred: false,
15988            e2e_defer_reason: None,
15989            progressed: false,
15990            vote_split: false,
15991            reconsideration: Vec::new(),
15992            verdict: None,
15993        }];
15994
15995        let mut runner = Runner {
15996            state,
15997            roles: ResolvedRoles {
15998                implementers: Vec::new(),
15999                judges: Vec::new(),
16000                reviewers: Vec::new(),
16001                fixer: None,
16002                conductor: conductor(),
16003                implementer_roster: Vec::new(),
16004                judge_roster: Vec::new(),
16005                reviewer_roster: Vec::new(),
16006            },
16007            sem: Arc::new(Semaphore::new(1)),
16008            pause: Pause::new(),
16009            interrupt: Pause::new(),
16010        };
16011
16012        let started = std::time::Instant::now();
16013        runner.gate().await.expect("gate");
16014        assert!(
16015            started.elapsed() < Duration::from_secs(1),
16016            "a gate with nothing to run must never wait on a lease it never needed"
16017        );
16018        assert!(
16019            runner.state.gate_ran,
16020            "zero commands is still a real, immediate attempt"
16021        );
16022        assert!(runner.state.gate.is_empty());
16023        assert_ne!(
16024            runner.state.status,
16025            RunStatus::Blocked,
16026            "must not read as resource-blocked on a lease it never asked for"
16027        );
16028    }
16029
16030    #[cfg(unix)]
16031    #[tokio::test]
16032    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
16033        crate::run::pin_test_home();
16034        let tmp = tempfile::tempdir().unwrap();
16035        let mut config = Config::default();
16036        config.disk.min_free_bytes = 0;
16037        let mut state = RunState::new(
16038            tmp.path().to_owned(),
16039            "main".into(),
16040            "base".into(),
16041            "verification evidence".into(),
16042            config,
16043        );
16044        let (outcomes, pids) = run_commands(
16045            &mut state,
16046            "gate",
16047            "gate",
16048            1,
16049            &["sh".into(), "-c".into()],
16050            &[
16051                "printf before-timeout; exec sleep 30".into(),
16052                "kill -TERM $$".into(),
16053                "exit 7".into(),
16054                "printf success".into(),
16055            ],
16056            tmp.path(),
16057            Duration::from_secs(1),
16058        )
16059        .await;
16060        wait_for_timed_out_children_to_die(&pids).await;
16061        assert_eq!(
16062            pids.len(),
16063            1,
16064            "unleased timeout retains its death-check PID"
16065        );
16066        assert_eq!(outcomes.len(), 4);
16067        assert_eq!(outcomes[0].code, None);
16068        assert!(outcomes[0].output_tail.contains("before-timeout"));
16069        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
16070        assert_eq!(outcomes[1].code, None);
16071        assert!(outcomes[1].output_tail.contains("command terminated:"));
16072        assert!(!outcomes[1].output_tail.contains("timed out"));
16073        assert_eq!(outcomes[2].code, Some(7));
16074        assert!(!outcomes[2].output_tail.contains("timed out"));
16075        assert_eq!(outcomes[3].code, Some(0));
16076        assert_eq!(outcomes[3].output_tail, "success");
16077        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
16078    }
16079
16080    /// The addendum's second gap: a `verify.gate` command running for real
16081    /// wall-clock time had nothing at all to show for it in `active` before
16082    /// `run_commands` learned to record it — a run could sit in `Gating` for
16083    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
16084    /// what was actually happening. Proven with a genuinely still-running
16085    /// command, not just a before/after check on the final state: a poller
16086    /// task reads the same `run.json` `gate()` is writing, the same way the
16087    /// phone or `magi show` would, while the shell command is still blocked
16088    /// on its own release marker.
16089    #[tokio::test]
16090    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
16091        crate::run::pin_test_home();
16092
16093        let tmp = tempfile::tempdir().expect("tempdir");
16094        let repo = tmp.path().join("repo");
16095        std::fs::create_dir_all(&repo).unwrap();
16096        init_repo(&repo);
16097
16098        let mut config = Config::default();
16099        config.verify.gate = vec![
16100            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
16101             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
16102                .to_owned(),
16103        ];
16104
16105        let mut state = RunState::new(
16106            repo.clone(),
16107            "main".to_owned(),
16108            "deadbeef".to_owned(),
16109            "task".to_owned(),
16110            config,
16111        );
16112        let run_id = state.id.clone();
16113        state.candidates = vec![Candidate {
16114            index: 0,
16115            label: 'A',
16116            agent: "alpha".to_owned(),
16117            branch: "does-not-exist".to_owned(),
16118            worktree: repo.clone(),
16119            summary: String::new(),
16120            stat: String::new(),
16121            files: 0,
16122            commits: 0,
16123            empty: false,
16124            failed: None,
16125            verified_noop: None,
16126            duration_ms: 0,
16127            folded: false,
16128        }];
16129        state.tally = Some(Tally {
16130            first_choice: BTreeMap::from([('A', 1)]),
16131            borda: BTreeMap::new(),
16132            winner: 'A',
16133            rankings: 1,
16134            unanimous_initial: true,
16135            deliberated: false,
16136            changed_votes: 0,
16137            unanimous_final: true,
16138            tie_break: None,
16139            judges: 0,
16140            present: 0,
16141            quorum: 0,
16142            met_quorum: true,
16143            uncontested: Some("only candidate A produced a change".to_owned()),
16144        });
16145        state.reviews = vec![ReviewRound {
16146            round: 1,
16147            head: "deadbeef".to_owned(),
16148            verified_head: None,
16149            verified_at: None,
16150            reviews: Vec::new(),
16151            e2e: Vec::new(),
16152            fix: None,
16153            blocking: 0,
16154            answered: 0,
16155            expected: 0,
16156            clean: true,
16157            verify_retried: false,
16158            e2e_deferred: false,
16159            e2e_defer_reason: None,
16160            progressed: false,
16161            vote_split: false,
16162            reconsideration: Vec::new(),
16163            verdict: None,
16164        }];
16165
16166        let mut runner = Runner {
16167            state,
16168            roles: ResolvedRoles {
16169                implementers: Vec::new(),
16170                judges: Vec::new(),
16171                reviewers: Vec::new(),
16172                fixer: None,
16173                conductor: conductor(),
16174                implementer_roster: Vec::new(),
16175                judge_roster: Vec::new(),
16176                reviewer_roster: Vec::new(),
16177            },
16178            sem: Arc::new(Semaphore::new(1)),
16179            pause: Pause::new(),
16180            interrupt: Pause::new(),
16181        };
16182
16183        let started_marker = repo.join("started.marker");
16184        let release_marker = repo.join("release.marker");
16185        let poller = tokio::spawn(async move {
16186            // Bounded so a regression that never records the task entry
16187            // fails this test in seconds instead of hanging the suite —
16188            // the same shape `a_park_requested_while_a_seat_is_mid_call_
16189            // does_not_cut_it_short` uses for the same reason.
16190            for _ in 0..100 {
16191                if started_marker.exists()
16192                    && let Ok(s) = crate::run::RunState::load(&run_id)
16193                    && let Some(a) = s.active.get("gate")
16194                {
16195                    std::fs::write(&release_marker, b"go").expect("release marker");
16196                    return Some(a.clone());
16197                }
16198                tokio::time::sleep(Duration::from_millis(50)).await;
16199            }
16200            None
16201        });
16202
16203        runner.gate().await.expect("gate");
16204        let captured = poller.await.expect("poller task");
16205        let captured = captured.expect(
16206            "the poller never saw a `gate` task entry in run.json while the command was \
16207             still blocked on its own release marker",
16208        );
16209
16210        assert_eq!(captured.task.as_deref(), Some("gate"));
16211        assert_eq!(captured.node, "gate");
16212        assert_eq!(captured.index, Some(1));
16213        assert_eq!(captured.total, Some(1));
16214        assert!(
16215            captured
16216                .command
16217                .as_deref()
16218                .is_some_and(|c| c.contains("started.marker")),
16219            "{captured:?}"
16220        );
16221
16222        assert!(
16223            runner.state.active.is_empty(),
16224            "the entry must be cleared once the command actually finished: {:?}",
16225            runner.state.active
16226        );
16227        assert!(runner.state.gate_ran);
16228        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16229    }
16230
16231    /// The hand-off over a blocking finding a reviewer rejected on leaves a
16232    /// record for `land`; one with only a Minor, or no reject, leaves none.
16233    #[tokio::test]
16234    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16235        use crate::verdict::{Finding, ReviewVote, Severity};
16236        crate::run::pin_test_home();
16237        let tmp = tempfile::tempdir().expect("tempdir");
16238        let repo = tmp.path().join("repo");
16239        std::fs::create_dir_all(&repo).unwrap();
16240        init_repo(&repo);
16241
16242        for (severity, vote, expect) in [
16243            (Severity::Major, ReviewVote::Reject, true),
16244            (Severity::Minor, ReviewVote::Reject, false),
16245            (Severity::Major, ReviewVote::Approve, false),
16246        ] {
16247            let mut round = review_round(false, 1, 1, 1, false, true);
16248            round.reviews = vec![ReviewRecord {
16249                reviewer: 1,
16250                agent: "alpha".to_owned(),
16251                summary: String::new(),
16252                findings: vec![Finding {
16253                    id: "R1-1-1".to_owned(),
16254                    severity,
16255                    file: None,
16256                    line: None,
16257                    title: "t".to_owned(),
16258                    detail: String::new(),
16259                }],
16260                vote: Some(vote),
16261                failed: None,
16262                duration_ms: 0,
16263                attempts: 0,
16264            }];
16265            let mut state = RunState::new(
16266                repo.clone(),
16267                "main".to_owned(),
16268                "deadbeef".to_owned(),
16269                "task".to_owned(),
16270                Config::default(),
16271            );
16272            state.reviews = vec![round];
16273            let mut runner = Runner {
16274                state,
16275                roles: ResolvedRoles {
16276                    implementers: Vec::new(),
16277                    judges: Vec::new(),
16278                    reviewers: Vec::new(),
16279                    fixer: None,
16280                    conductor: conductor(),
16281                    implementer_roster: Vec::new(),
16282                    judge_roster: Vec::new(),
16283                    reviewer_roster: Vec::new(),
16284                },
16285                sem: Arc::new(Semaphore::new(1)),
16286                pause: Pause::new(),
16287                interrupt: Pause::new(),
16288            };
16289            let shell = runner.state.config.shell();
16290            runner
16291                .stop_reviewing("round budget spent", &shell, &repo)
16292                .await
16293                .expect("stop_reviewing");
16294            assert_eq!(runner.state.status, RunStatus::Gating);
16295            assert_eq!(
16296                runner.state.contested_handoff.is_some(),
16297                expect,
16298                "{severity:?} + {vote:?}"
16299            );
16300        }
16301    }
16302
16303    /// The shape the incident this whole fix responds to actually had: the
16304    /// round budget spent, the last round's own e2e blocked on the shared
16305    /// build cache (held here by a live pid — this test process — exactly
16306    /// `cache`'s own unit tests' pattern for "another owner, still alive"
16307    /// without forking a process). `stop_reviewing` must retry it — not
16308    /// silently leave the round looking untouched (the catch-up-only half of
16309    /// the bug), and not read the contention as a red `e2e` and block the
16310    /// run on it (the other half). Called directly, the same way
16311    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
16312    /// above exercises `gate`, so this never needs a real cargo build to
16313    /// reach: the lease is never released, so `with_cache_lease` never gets
16314    /// past acquiring it into anything that would need a real workspace.
16315    #[tokio::test]
16316    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16317        crate::run::pin_test_home();
16318        let home = crate::run::home();
16319
16320        let tmp = tempfile::tempdir().expect("tempdir");
16321        let repo = tmp.path().join("repo");
16322        std::fs::create_dir_all(&repo).unwrap();
16323        init_repo(&repo);
16324        let head = crate::git::rev_parse(&repo, "HEAD")
16325            .await
16326            .expect("rev-parse");
16327        // Unique to this test, so holding its lease cannot collide with
16328        // another test sharing the same process-wide `home`.
16329        let cache_dir = tmp.path().join("target");
16330
16331        let mut config = Config::default();
16332        config.verify.e2e = vec![format!(
16333            "CARGO_TARGET_DIR='{}' test -f README.md",
16334            cache_dir.display()
16335        )];
16336        config.graph.review_rounds = 1;
16337        // Bounded so a regression that does start waiting fails the test in
16338        // seconds, not hangs it.
16339        config.graph.timeout_verify = Some(2);
16340        config.graph.timeout_resource_wait = 2;
16341
16342        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16343        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16344            .expect("no io error acquiring directly")
16345        {
16346            crate::cache::AcquireOutcome::Acquired(g) => g,
16347            crate::cache::AcquireOutcome::Busy(b) => {
16348                panic!("expected the direct acquire to win the lease first: {b:?}")
16349            }
16350        };
16351
16352        let mut state = RunState::new(
16353            repo.clone(),
16354            "main".to_owned(),
16355            head.clone(),
16356            "task".to_owned(),
16357            config,
16358        );
16359        state.candidates = vec![Candidate {
16360            index: 0,
16361            label: 'A',
16362            agent: "alpha".to_owned(),
16363            branch: "does-not-exist".to_owned(),
16364            worktree: repo.clone(),
16365            summary: String::new(),
16366            stat: String::new(),
16367            files: 0,
16368            commits: 0,
16369            empty: false,
16370            failed: None,
16371            verified_noop: None,
16372            duration_ms: 0,
16373            folded: false,
16374        }];
16375        state.tally = Some(Tally {
16376            first_choice: BTreeMap::from([('A', 1)]),
16377            borda: BTreeMap::new(),
16378            winner: 'A',
16379            rankings: 1,
16380            unanimous_initial: true,
16381            deliberated: false,
16382            changed_votes: 0,
16383            unanimous_final: true,
16384            tie_break: None,
16385            judges: 0,
16386            present: 0,
16387            quorum: 0,
16388            met_quorum: true,
16389            uncontested: Some("only candidate A produced a change".to_owned()),
16390        });
16391        // The round budget's last round, deferred: `needs_catchup_run`'s
16392        // other trigger. `stop_reviewing`'s retry machinery must treat this
16393        // exactly like a resource-blocked attempt once it actually runs.
16394        state.reviews = vec![ReviewRound {
16395            round: 1,
16396            head: head.clone(),
16397            verified_head: None,
16398            verified_at: None,
16399            reviews: Vec::new(),
16400            e2e: Vec::new(),
16401            fix: None,
16402            blocking: 1,
16403            answered: 1,
16404            expected: 1,
16405            clean: false,
16406            verify_retried: false,
16407            e2e_deferred: true,
16408            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16409            progressed: false,
16410            vote_split: false,
16411            reconsideration: Vec::new(),
16412            verdict: None,
16413        }];
16414
16415        let mut runner = Runner {
16416            state,
16417            roles: ResolvedRoles {
16418                implementers: Vec::new(),
16419                judges: Vec::new(),
16420                reviewers: Vec::new(),
16421                fixer: None,
16422                conductor: conductor(),
16423                implementer_roster: Vec::new(),
16424                judge_roster: Vec::new(),
16425                reviewer_roster: Vec::new(),
16426            },
16427            sem: Arc::new(Semaphore::new(1)),
16428            pause: Pause::new(),
16429            interrupt: Pause::new(),
16430        };
16431
16432        let shell = runner.state.config.shell();
16433        runner
16434            .stop_reviewing("round budget spent", &shell, &repo)
16435            .await
16436            .expect("stop_reviewing");
16437
16438        let last = runner.state.reviews.last().expect("round record");
16439        assert_eq!(
16440            last.e2e_status(),
16441            E2eStatus::ResourceBlocked,
16442            "the shared cache is still held; the attempt must read as blocked, not deferred or \
16443             failed: {last:?}"
16444        );
16445        assert_eq!(
16446            last.verified_head.as_deref(),
16447            Some(head.as_str()),
16448            "which commit this attempt targeted is known even though nothing finished checking \
16449             it"
16450        );
16451        let first_attempt_at = last
16452            .verified_at
16453            .expect("when this attempt ran is known too");
16454        assert_ne!(
16455            runner.state.status,
16456            RunStatus::Blocked,
16457            "contention is evidence about the machine, not the patch — it must not settle the \
16458             run as blocked: {:?}",
16459            runner.state.status
16460        );
16461        assert!(
16462            !runner
16463                .state
16464                .events
16465                .iter()
16466                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16467            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16468            runner.state.events
16469        );
16470
16471        // The cache is still held: a later reentry must retry the same
16472        // round's verification again — not leave it looking exactly as
16473        // untouched as the first blocked attempt, which is indistinguishable
16474        // from never having tried again at all.
16475        runner
16476            .stop_reviewing("round budget spent", &shell, &repo)
16477            .await
16478            .expect("stop_reviewing retry");
16479        assert_eq!(
16480            runner.state.reviews.len(),
16481            1,
16482            "no new round was started: {:?}",
16483            runner.state.reviews
16484        );
16485        let last = runner.state.reviews.last().expect("round record");
16486        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16487        assert!(
16488            last.verified_at.expect("still known") > first_attempt_at,
16489            "a second reentry must be a fresh attempt, not a stale copy of the first"
16490        );
16491        assert_ne!(runner.state.status, RunStatus::Blocked);
16492
16493        held.release();
16494    }
16495
16496    #[cfg(unix)]
16497    #[tokio::test]
16498    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
16499        ask_test_home();
16500        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
16501            let tmp = tempfile::tempdir().unwrap();
16502            let repo = tmp.path().join("repo");
16503            std::fs::create_dir(&repo).unwrap();
16504            init_repo(&repo);
16505            let run_git = |args: &[&str]| {
16506                let out = std::process::Command::new("git")
16507                    .args(args)
16508                    .current_dir(&repo)
16509                    .output()
16510                    .unwrap();
16511                assert!(
16512                    out.status.success(),
16513                    "{args:?}: {}",
16514                    String::from_utf8_lossy(&out.stderr)
16515                );
16516            };
16517            let origin = tmp.path().join("origin.git");
16518            run_git(&["init", "--bare", origin.to_str().unwrap()]);
16519            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
16520            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
16521            run_git(&["add", "spec.rs"]);
16522            run_git(&["commit", "-m", "initial spec"]);
16523            run_git(&["push", "origin", "main"]);
16524            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16525            run_git(&["checkout", "-b", "fix/review"]);
16526            std::fs::write(
16527                repo.join("check.rs"),
16528                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
16529            )
16530            .unwrap();
16531            run_git(&["add", "check.rs"]);
16532            run_git(&["commit", "-m", "use spec"]);
16533            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
16534            run_git(&["checkout", "main"]);
16535            std::fs::write(
16536                repo.join("spec.rs"),
16537                "struct AgentSpec { max_turn_secs: u64 }\n",
16538            )
16539            .unwrap();
16540            run_git(&["add", "spec.rs"]);
16541            run_git(&["commit", "-m", "extend spec"]);
16542            run_git(&["push", "origin", "main"]);
16543            run_git(&["checkout", "fix/review"]);
16544
16545            let mut runner = runner_at(RunStatus::Reviewing);
16546            runner.state.repo = repo.clone();
16547            runner.state.base_commit = base;
16548            runner.state.pr_head = Some("fix/review".into());
16549            runner.state.config.graph.review_rounds = max_rounds;
16550            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16551            runner.state.config.graph.sessions = false;
16552            runner.state.config.disk.min_free_bytes = 0;
16553            runner.state.config.verify.e2e =
16554                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
16555            runner.state.candidates = vec![Candidate {
16556                index: 0,
16557                label: 'A',
16558                agent: "fixture".into(),
16559                branch: "fix/review".into(),
16560                worktree: repo.clone(),
16561                summary: String::new(),
16562                stat: String::new(),
16563                files: 1,
16564                commits: 1,
16565                empty: false,
16566                failed: None,
16567                verified_noop: None,
16568                duration_ms: 0,
16569                folded: false,
16570            }];
16571            runner.state.tally = Some(Tally {
16572                first_choice: BTreeMap::new(),
16573                borda: BTreeMap::new(),
16574                winner: 'A',
16575                rankings: 0,
16576                unanimous_initial: true,
16577                deliberated: false,
16578                changed_votes: 0,
16579                unanimous_final: true,
16580                tie_break: None,
16581                judges: 0,
16582                present: 0,
16583                quorum: 0,
16584                met_quorum: true,
16585                uncontested: Some("review-only".into()),
16586            });
16587            let mut last = review_round(false, 0, 1, 1, false, false);
16588            last.head = reviewed.clone();
16589            last.e2e_deferred = true;
16590            last.e2e[0].resource_blocked = true;
16591            last.e2e[0].output_tail = "waiting for the shared build cache".into();
16592            if max_rounds == 1 {
16593                last.e2e.clear();
16594            }
16595            runner.state.reviews = vec![last];
16596            let calls = tmp.path().join("fix-calls");
16597            let mut fixer = spec("repair");
16598            fixer
16599                .env
16600                .insert("FIX_CALLS".into(), calls.display().to_string());
16601            fixer.command = vec!["sh".into(), "-c".into(), r#"
16602                echo fix >> "$FIX_CALLS"
16603                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
16604                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
16605                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
16606                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
16607            "#.into()];
16608            let mut reviewer = spec("reviewer");
16609            reviewer.command = vec![
16610                "sh".into(),
16611                "-c".into(),
16612                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
16613                    .into(),
16614            ];
16615            runner.roles.fixer = Some(vec![fixer]);
16616            runner.roles.reviewers = vec![reviewer];
16617
16618            runner.sync_to_base().await.unwrap();
16619            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
16620            assert_ne!(rebased, reviewed);
16621            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
16622            if reentry {
16623                // Simulate a crash after catch-up evidence was saved but before
16624                // dispatching the fixer. No dedicated pending-repair field exists.
16625                let shell = runner.state.config.shell();
16626                runner
16627                    .stop_unresolved_review("deferred verification", &shell, &repo)
16628                    .await
16629                    .unwrap();
16630                assert!(runner.catchup_repairable());
16631                assert!(!calls.exists());
16632                runner.state = RunState::load(&runner.state.id).unwrap();
16633                assert!(runner.catchup_repairable());
16634            }
16635            runner.review_loop().await.unwrap();
16636            let first = &runner.state.reviews[0];
16637            assert_eq!(first.head, reviewed);
16638            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
16639            assert!(
16640                first.verify_retried,
16641                "a reproduced build failure gets exactly one retry: {:?}",
16642                first.e2e
16643            );
16644            assert!(first.e2e[0].output_tail.contains("E0063"));
16645            assert!(!runner.catchup_repairable());
16646            if max_rounds == 2 {
16647                assert_eq!(
16648                    runner.state.status,
16649                    RunStatus::Gating,
16650                    "{}",
16651                    crate::report::run(&runner.state)
16652                );
16653                assert_eq!(runner.state.reviews.len(), 2);
16654                assert!(first.fix.as_ref().unwrap().committed);
16655                assert!(runner.state.reviews[1].clean);
16656                assert_eq!(
16657                    runner
16658                        .state
16659                        .reviews
16660                        .iter()
16661                        .filter(|r| r.fix.is_some())
16662                        .count(),
16663                    1
16664                );
16665                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
16666                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
16667            } else {
16668                assert_eq!(runner.state.status, RunStatus::Blocked);
16669                assert!(first.fix.is_none());
16670                assert!(!calls.exists());
16671                let recovery = crate::recovery::assess(
16672                    "prior",
16673                    Some(&runner.state),
16674                    &runner.landing_base(),
16675                    crate::recovery::Relation::Diverged,
16676                );
16677                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
16678                let reasons = recovery.reasons.join("\n");
16679                assert!(
16680                    reasons.contains(&reviewed) && reasons.contains(&rebased),
16681                    "{reasons}"
16682                );
16683                assert!(
16684                    reasons.contains("base sync to")
16685                        && reasons.contains("additional review is required"),
16686                    "{reasons}"
16687                );
16688            }
16689        }
16690    }
16691
16692    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
16693    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
16694    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
16695    /// own top-of-function fast path (`review_conclusion`) correctly reads
16696    /// this shape as `None` rather than guessing `Blocked`, and the loop's
16697    /// own `for` range is empty once the round budget is spent, so
16698    /// `review_loop` must retry the check itself rather than silently doing
16699    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
16700    /// above exercises directly, but through `review_loop`'s own entry point
16701    /// this time, proving the wiring between the two rather than just the
16702    /// retry logic in isolation.
16703    #[tokio::test]
16704    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16705        crate::run::pin_test_home();
16706        let home = crate::run::home();
16707
16708        let tmp = tempfile::tempdir().expect("tempdir");
16709        let repo = tmp.path().join("repo");
16710        std::fs::create_dir_all(&repo).unwrap();
16711        init_repo(&repo);
16712        let head = crate::git::rev_parse(&repo, "HEAD")
16713            .await
16714            .expect("rev-parse");
16715        let cache_dir = tmp.path().join("target");
16716
16717        let mut config = Config::default();
16718        config.verify.e2e = vec![format!(
16719            "CARGO_TARGET_DIR='{}' test -f README.md",
16720            cache_dir.display()
16721        )];
16722        config.graph.review_rounds = 1;
16723        config.graph.timeout_verify = Some(2);
16724        config.graph.timeout_resource_wait = 2;
16725
16726        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16727        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16728            .expect("no io error acquiring directly")
16729        {
16730            crate::cache::AcquireOutcome::Acquired(g) => g,
16731            crate::cache::AcquireOutcome::Busy(b) => {
16732                panic!("expected the direct acquire to win the lease first: {b:?}")
16733            }
16734        };
16735
16736        let mut state = RunState::new(
16737            repo.clone(),
16738            "main".to_owned(),
16739            head.clone(),
16740            "task".to_owned(),
16741            config,
16742        );
16743        state.candidates = vec![Candidate {
16744            index: 0,
16745            label: 'A',
16746            agent: "alpha".to_owned(),
16747            branch: "does-not-exist".to_owned(),
16748            worktree: repo.clone(),
16749            summary: String::new(),
16750            stat: String::new(),
16751            files: 0,
16752            commits: 0,
16753            empty: false,
16754            failed: None,
16755            verified_noop: None,
16756            duration_ms: 0,
16757            folded: false,
16758        }];
16759        state.tally = Some(Tally {
16760            first_choice: BTreeMap::from([('A', 1)]),
16761            borda: BTreeMap::new(),
16762            winner: 'A',
16763            rankings: 1,
16764            unanimous_initial: true,
16765            deliberated: false,
16766            changed_votes: 0,
16767            unanimous_final: true,
16768            tie_break: None,
16769            judges: 0,
16770            present: 0,
16771            quorum: 0,
16772            met_quorum: true,
16773            uncontested: Some("only candidate A produced a change".to_owned()),
16774        });
16775        // The exact shape a prior process's `stop_reviewing` would have left
16776        // on disk: the round budget's last round, a real attempt already
16777        // made and already resource-blocked.
16778        state.reviews = vec![ReviewRound {
16779            round: 1,
16780            head: head.clone(),
16781            verified_head: Some(head.clone()),
16782            verified_at: Some(jiff::Timestamp::now()),
16783            reviews: Vec::new(),
16784            e2e: vec![CommandOutcome {
16785                command: format!(
16786                    "CARGO_TARGET_DIR='{}' test -f README.md",
16787                    cache_dir.display()
16788                ),
16789                code: None,
16790                output_tail: "waiting for the shared build cache".to_owned(),
16791                duration_ms: 0,
16792                resource_blocked: true,
16793            }],
16794            fix: None,
16795            blocking: 1,
16796            answered: 1,
16797            expected: 1,
16798            clean: false,
16799            verify_retried: false,
16800            e2e_deferred: false,
16801            e2e_defer_reason: None,
16802            progressed: false,
16803            vote_split: false,
16804            reconsideration: Vec::new(),
16805            verdict: None,
16806        }];
16807
16808        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16809        let mut runner = Runner {
16810            state,
16811            roles: ResolvedRoles {
16812                implementers: Vec::new(),
16813                judges: Vec::new(),
16814                reviewers: Vec::new(),
16815                fixer: None,
16816                conductor: conductor(),
16817                implementer_roster: Vec::new(),
16818                judge_roster: Vec::new(),
16819                reviewer_roster: Vec::new(),
16820            },
16821            sem: Arc::new(Semaphore::new(1)),
16822            pause: Pause::new(),
16823            interrupt: Pause::new(),
16824        };
16825
16826        // The lease is still held throughout, so this reentry's own retry is
16827        // also contended — proving `review_loop` actually tried again (not
16828        // that it happened to succeed) is what the timestamp comparison
16829        // below is for.
16830        runner.review_loop().await.expect("review_loop");
16831
16832        assert_eq!(
16833            runner.state.reviews.len(),
16834            1,
16835            "no new round was started on top of the unresolved one: {:?}",
16836            runner.state.reviews
16837        );
16838        let last = &runner.state.reviews[0];
16839        assert_eq!(
16840            last.e2e_status(),
16841            E2eStatus::ResourceBlocked,
16842            "still contended: {last:?}"
16843        );
16844        assert!(
16845            last.verified_at.expect("still known") > first_attempt_at,
16846            "review_loop must have actually retried the check, not left it exactly as found"
16847        );
16848        assert_ne!(
16849            runner.state.status,
16850            RunStatus::Blocked,
16851            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16852            runner.state.status
16853        );
16854
16855        held.release();
16856    }
16857
16858    #[tokio::test]
16859    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
16860        crate::run::pin_test_home();
16861        for status in [RunStatus::Blocked, RunStatus::Gating] {
16862            let mut runner = runner_at(status);
16863            runner.state.config.graph.land = true;
16864            runner.state.gate_ran = true;
16865            runner.state.merge = Some(MergeOutcome {
16866                mode: MergeMode::Pr,
16867                ok: false,
16868                detail: "the fixer produced no commit".into(),
16869                empty: false,
16870            });
16871            let url = "https://example.invalid/o/r/pull/1";
16872            runner.state.pr = Some(crate::run::PrRecord {
16873                url: url.into(),
16874                number: 1,
16875                state: "open".into(),
16876                checks: "red".into(),
16877                round: 1,
16878                rounds: 1,
16879                red_at_merge: Vec::new(),
16880            });
16881            runner.state.save().unwrap();
16882            // The nonexistent repository makes observation fail before a forge
16883            // command can run. We still must reach land, rather than silently
16884            // return from the graph with its old gate and merge records.
16885            runner.execute_graph().await.unwrap();
16886            assert_eq!(runner.state.status, RunStatus::Blocked);
16887            assert!(runner.state.land_sync_pending);
16888            assert!(runner.state.events.iter().any(|event|
16889                event.node == "land" && event.message == format!("watching {url}")));
16890            assert_eq!(
16891                runner.state.merge.as_ref().unwrap().detail,
16892                "the fixer produced no commit"
16893            );
16894        }
16895    }
16896
16897    #[tokio::test]
16898    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16899        crate::run::pin_test_home();
16900        let tmp = tempfile::tempdir().expect("tempdir");
16901        let repo = tmp.path().join("repo");
16902        std::fs::create_dir_all(&repo).unwrap();
16903        init_repo(&repo);
16904
16905        let mut config = Config::default();
16906        config.merge.mode = MergeMode::Pr;
16907        config.graph.land = true;
16908        config.graph.land_approval = false;
16909
16910        let mut state = RunState::new(
16911            repo.clone(),
16912            "main".to_owned(),
16913            "deadbeef".to_owned(),
16914            "task".to_owned(),
16915            config,
16916        );
16917        state.candidates = vec![Candidate {
16918            index: 0,
16919            label: 'A',
16920            agent: "alpha".to_owned(),
16921            branch: "does-not-exist".to_owned(),
16922            worktree: repo.clone(),
16923            summary: String::new(),
16924            stat: String::new(),
16925            files: 0,
16926            commits: 0,
16927            empty: false,
16928            failed: None,
16929            verified_noop: None,
16930            duration_ms: 0,
16931            folded: false,
16932        }];
16933        state.tally = Some(Tally {
16934            first_choice: BTreeMap::from([('A', 1)]),
16935            borda: BTreeMap::new(),
16936            winner: 'A',
16937            rankings: 1,
16938            unanimous_initial: true,
16939            deliberated: false,
16940            changed_votes: 0,
16941            unanimous_final: true,
16942            tie_break: None,
16943            judges: 0,
16944            present: 0,
16945            quorum: 0,
16946            met_quorum: true,
16947            uncontested: Some("only candidate A produced a change".to_owned()),
16948        });
16949        state.reviews = vec![ReviewRound {
16950            round: 1,
16951            head: "deadbeef".to_owned(),
16952            verified_head: None,
16953            verified_at: None,
16954            reviews: Vec::new(),
16955            e2e: Vec::new(),
16956            fix: None,
16957            blocking: 0,
16958            answered: 0,
16959            expected: 0,
16960            clean: true,
16961            verify_retried: false,
16962            e2e_deferred: false,
16963            e2e_defer_reason: None,
16964            progressed: false,
16965            vote_split: false,
16966            reconsideration: Vec::new(),
16967            verdict: None,
16968        }];
16969        state.gate = vec![CommandOutcome {
16970            command: "test".to_owned(),
16971            code: Some(0),
16972            output_tail: String::new(),
16973            duration_ms: 0,
16974            resource_blocked: false,
16975        }];
16976        state.gate_ran = true;
16977        // A first pass through `merge` already pushed and opened this pull
16978        // request; `status` is `Landing` because a previous call into `land`
16979        // parked or was interrupted before it reached a terminal outcome.
16980        state.status = RunStatus::Landing;
16981        state.merge = Some(MergeOutcome {
16982            mode: MergeMode::Pr,
16983            ok: true,
16984            detail: "https://example.invalid/x/y/pull/1".to_owned(),
16985            empty: false,
16986        });
16987
16988        // The Landing-resume shortcut calls `run_land` directly rather than
16989        // through `merge`, which is exactly the call site that used to skip
16990        // `settle_questions` - see the fixture below.
16991        ask_test_home();
16992        // A real resume loads a persisted run; driver ownership is checked
16993        // against that record before the Landing shortcut is entered.
16994        state.save().expect("save landing run");
16995        let store = ask::Questions::open();
16996        let q = ask_open_question(&store, &state.id);
16997
16998        let mut runner = Runner {
16999            state,
17000            roles: ResolvedRoles {
17001                implementers: Vec::new(),
17002                judges: Vec::new(),
17003                reviewers: Vec::new(),
17004                fixer: None,
17005                conductor: conductor(),
17006                implementer_roster: Vec::new(),
17007                judge_roster: Vec::new(),
17008                reviewer_roster: Vec::new(),
17009            },
17010            sem: Arc::new(Semaphore::new(1)),
17011            pause: Pause::new(),
17012            interrupt: Pause::new(),
17013        };
17014
17015        // `execute`, not `merge` directly: the Landing-resume shortcut lives
17016        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
17017        // exactly because `review_loop` would otherwise clobber the marker
17018        // first.
17019        runner.execute().await.expect("execute");
17020
17021        assert_eq!(
17022            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17023            Some("https://example.invalid/x/y/pull/1"),
17024            "reentry must not push again or open a second pull request over the \
17025             one `land` is already watching"
17026        );
17027        assert_ne!(
17028            runner.state.status,
17029            RunStatus::Landing,
17030            "land could not actually reach the fake pull request, so it must \
17031             have given up rather than left the run silently parked forever"
17032        );
17033        // `land` could not reach the fake pull request, so it gave up into
17034        // `Blocked` - still resumable, so the question must not have been
17035        // swept just because this branch now also calls `settle_questions`.
17036        assert_eq!(runner.state.status, RunStatus::Blocked);
17037        assert!(
17038            store.get(&q.id).unwrap().status.open(),
17039            "Blocked is still alive; settle_questions must have been a no-op here"
17040        );
17041    }
17042
17043    fn state_with_round(round: ReviewRound) -> RunState {
17044        let mut s = RunState::new(
17045            PathBuf::from("/repo"),
17046            "main".to_owned(),
17047            "abc1234".to_owned(),
17048            "add retries".to_owned(),
17049            Config::default(),
17050        );
17051        s.reviews = vec![round];
17052        s
17053    }
17054
17055    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
17056        crate::verdict::Finding {
17057            id: id.to_owned(),
17058            severity,
17059            file: None,
17060            line: None,
17061            title: title.to_owned(),
17062            detail: String::new(),
17063        }
17064    }
17065
17066    #[test]
17067    fn pr_body_names_open_findings_and_declined_ones() {
17068        let round = ReviewRound {
17069            round: 2,
17070            head: "deadbee".to_owned(),
17071            verified_head: None,
17072            verified_at: None,
17073            reviews: vec![ReviewRecord {
17074                attempts: 0,
17075                reviewer: 1,
17076                agent: "alpha".to_owned(),
17077                summary: String::new(),
17078                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
17079                vote: None,
17080                failed: None,
17081                duration_ms: 0,
17082            }],
17083            e2e: vec![CommandOutcome {
17084                command: "cargo test".to_owned(),
17085                code: Some(0),
17086                output_tail: String::new(),
17087                duration_ms: 0,
17088                resource_blocked: false,
17089            }],
17090            verify_retried: false,
17091            e2e_deferred: false,
17092            e2e_defer_reason: None,
17093            fix: Some(FixRecord {
17094                agent: "alpha".to_owned(),
17095                addressed: Vec::new(),
17096                rejected: vec![crate::verdict::Rejection {
17097                    id: "R1-1-1".to_owned(),
17098                    why: "not reachable from any caller".to_owned(),
17099                }],
17100                notes: String::new(),
17101                committed: true,
17102                failed: None,
17103                duration_ms: 0,
17104                continuation: None,
17105            }),
17106            blocking: 0,
17107            answered: 1,
17108            expected: 1,
17109            clean: false,
17110            progressed: true,
17111            vote_split: false,
17112            reconsideration: Vec::new(),
17113            verdict: None,
17114        };
17115        let state = state_with_round(round);
17116        let body = pr_message(&state, 'A').body;
17117
17118        assert!(body.contains("add retries"), "the task must still be there");
17119        assert!(body.contains("R2-1-1"), "{body}");
17120        assert!(!body.contains("unused import"), "{body}");
17121        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
17122        assert!(
17123            !body.contains("not reachable from any caller"),
17124            "the reason it was declined: {body}"
17125        );
17126    }
17127
17128    #[test]
17129    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
17130        let round = ReviewRound {
17131            round: 1,
17132            head: "deadbee".to_owned(),
17133            verified_head: None,
17134            verified_at: None,
17135            reviews: vec![ReviewRecord {
17136                attempts: 0,
17137                reviewer: 1,
17138                agent: "alpha".to_owned(),
17139                summary: String::new(),
17140                findings: Vec::new(),
17141                vote: None,
17142                failed: None,
17143                duration_ms: 0,
17144            }],
17145            e2e: Vec::new(),
17146            verify_retried: false,
17147            e2e_deferred: false,
17148            e2e_defer_reason: None,
17149            fix: None,
17150            blocking: 0,
17151            answered: 1,
17152            expected: 1,
17153            clean: true,
17154            progressed: false,
17155            vote_split: false,
17156            reconsideration: Vec::new(),
17157            verdict: None,
17158        };
17159        let state = state_with_round(round);
17160        let body = pr_message(&state, 'A').body;
17161        assert!(!body.contains("Open review findings"), "{body}");
17162        assert!(!body.contains("Declined"), "{body}");
17163    }
17164
17165    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
17166        let mut state = RunState::new(
17167            PathBuf::from("/repo"),
17168            "main".to_owned(),
17169            "abc1234".to_owned(),
17170            instruction.to_owned(),
17171            Config::default(),
17172        );
17173        state.candidates.push(Candidate {
17174            index: 0,
17175            label: 'A',
17176            agent: "alpha".to_owned(),
17177            branch: "magi/x/A".to_owned(),
17178            worktree: PathBuf::from("/wt"),
17179            summary: summary.to_owned(),
17180            stat: String::new(),
17181            files: 1,
17182            commits: 1,
17183            empty: false,
17184            failed: None,
17185            verified_noop: None,
17186            folded: false,
17187            duration_ms: 0,
17188        });
17189        state
17190    }
17191
17192    fn review_state(subjects: &[&str]) -> RunState {
17193        let mut state = state_with_summary(
17194            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
17195            "",
17196        );
17197        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17198        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
17199        state
17200    }
17201
17202    /// A branch rebased onto a main that moved past the recorded
17203    /// `base_commit` is titled from its own first commit, never main's.
17204    #[tokio::test]
17205    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
17206        ask_test_home();
17207        let tmp = tempfile::tempdir().unwrap();
17208        let repo = tmp.path().join("repo");
17209        std::fs::create_dir_all(&repo).unwrap();
17210        init_repo(&repo);
17211        let origin = tmp.path().join("origin.git");
17212        let g = |dir: &Path, args: &[&str]| {
17213            let out = std::process::Command::new("git")
17214                .args(args)
17215                .current_dir(dir)
17216                .quiet()
17217                .output()
17218                .expect("spawn git");
17219            assert!(
17220                out.status.success(),
17221                "git {args:?}: {}",
17222                String::from_utf8_lossy(&out.stderr)
17223            );
17224        };
17225        g(
17226            tmp.path(),
17227            &[
17228                "clone",
17229                "--bare",
17230                "-q",
17231                repo.to_str().unwrap(),
17232                origin.to_str().unwrap(),
17233            ],
17234        );
17235        g(
17236            &repo,
17237            &["remote", "add", "origin", origin.to_str().unwrap()],
17238        );
17239        let c1 = git::rev_parse(&repo, "main").await.unwrap();
17240
17241        // Main moves on; the branch is built on top of the new main.
17242        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
17243        g(&repo, &["add", "-A"]);
17244        g(
17245            &repo,
17246            &["commit", "-q", "-m", "chore(deps): update a crate"],
17247        );
17248        g(&repo, &["push", "-q", "origin", "main"]);
17249        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
17250        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17251        g(&repo, &["add", "-A"]);
17252        g(
17253            &repo,
17254            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
17255        );
17256        g(&repo, &["checkout", "-q", "main"]);
17257
17258        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
17259        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
17260        // Without a readable tracking ref the recorded base's merge base is used.
17261        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
17262        assert_eq!(fallback, c1);
17263
17264        let mut state = review_state(&[
17265            "chore(deps): update a crate",
17266            "fix(daemon): apply a chosen action",
17267        ]);
17268        state.repo = repo.clone();
17269        state.base_branch = "main".to_owned();
17270        state.base_commit = c1;
17271        refresh_reviewed_commits(&mut state, "feat/own").await;
17272        assert_eq!(
17273            state.reviewed_commits,
17274            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
17275        );
17276        assert_eq!(
17277            review_title(&state).as_deref(),
17278            Some("fix(daemon): apply a chosen action")
17279        );
17280        assert_eq!(
17281            leaked_subjects(&state, "feat/own").await,
17282            Some(vec!["chore(deps): update a crate".to_owned()])
17283        );
17284        // A stale tracking ref is still used when the fetch fails, but the
17285        // leak list is withheld.
17286        state.config.merge.remote = "nowhere".to_owned();
17287        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
17288    }
17289
17290    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
17291    /// with `origin/main` one commit behind it, and a summarizing agent that
17292    /// prints whatever `reply` holds.
17293    struct SummaryFixture {
17294        _tmp: tempfile::TempDir,
17295        repo: PathBuf,
17296        reply: PathBuf,
17297        runner: Runner,
17298    }
17299
17300    fn summary_fixture() -> SummaryFixture {
17301        ask_test_home();
17302        let tmp = tempfile::tempdir().unwrap();
17303        let repo = tmp.path().join("repo");
17304        std::fs::create_dir_all(&repo).unwrap();
17305        init_repo(&repo);
17306        let origin = tmp.path().join("origin.git");
17307        let g = |args: &[&str]| {
17308            let out = std::process::Command::new("git")
17309                .args(args)
17310                .current_dir(&repo)
17311                .quiet()
17312                .output()
17313                .expect("spawn git");
17314            assert!(
17315                out.status.success(),
17316                "git {args:?}: {}",
17317                String::from_utf8_lossy(&out.stderr)
17318            );
17319        };
17320        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
17321        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
17322        g(&["push", "-q", "origin", "main"]);
17323        g(&["checkout", "-q", "-b", "feat/own"]);
17324        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17325        g(&["add", "-A"]);
17326        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
17327        let reply = tmp.path().join("reply.txt");
17328        std::fs::write(
17329            &reply,
17330            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
17331        )
17332        .unwrap();
17333        let mut runner = runner_at(RunStatus::Gating);
17334        let mut state = review_state(&["feat(own): add the own file"]);
17335        state.repo = repo.clone();
17336        state.base_branch = "main".to_owned();
17337        state.candidates[0].branch = "feat/own".to_owned();
17338        state.candidates[0].worktree = repo.clone();
17339        let mut author = spec("alpha");
17340        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17341        author
17342            .env
17343            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17344        state.config.agents = vec![author];
17345        runner.state = state;
17346        SummaryFixture {
17347            _tmp: tmp,
17348            repo,
17349            reply,
17350            runner,
17351        }
17352    }
17353
17354    async fn posted(f: &mut SummaryFixture) -> PrMessage {
17355        let winner = f.runner.state.candidates[0].clone();
17356        f.runner
17357            .guarded_pr_message(&winner, None, true)
17358            .await
17359            .unwrap()
17360            .expect("an English description never parks")
17361    }
17362
17363    fn clean_round_with_minor_findings() -> ReviewRound {
17364        let mut round = passing_round(2, None);
17365        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17366        round.reviews = vec![ReviewRecord {
17367            attempts: 0,
17368            reviewer: 1,
17369            agent: "alpha".into(),
17370            summary: "内部レビュー".into(),
17371            findings: vec![
17372                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17373                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17374            ],
17375            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17376            failed: None,
17377            duration_ms: 0,
17378        }];
17379        round
17380    }
17381
17382    #[tokio::test]
17383    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17384        for review_only in [false, true] {
17385            let mut f = summary_fixture();
17386            if !review_only {
17387                f.runner.state.candidates[0].agent = "alpha".into();
17388            }
17389            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17390            assert!(f.runner.state.reviews[0].clean);
17391            assert!(f.runner.state.reviews[0].fix.is_none());
17392            assert!(f.runner.state.open_findings().is_empty());
17393            assert_eq!(
17394                crate::pr_outcome::required_finding_ids(&f.runner.state),
17395                ["R2-1-1", "R2-1-2"]
17396            );
17397            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17398            let evidence = crate::pr_outcome::summary_evidence(
17399                &f.runner.state,
17400                crate::pr_outcome::FinalHead::At(&head),
17401            );
17402            assert!(
17403                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17404            );
17405            // The ordinary reply omits both IDs: no cached description is adopted.
17406            let omitted = posted(&mut f).await;
17407            assert!(omitted.body.contains("No description"));
17408            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17409            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17410            assert!(f.runner.state.change_summary.is_none());
17411            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17412            let described = posted(&mut f).await;
17413            assert!(described.body.contains("improve documentation for callers"));
17414            assert!(described.body.contains("remaining edge case"));
17415            assert!(described.body.contains("## Open review findings"));
17416            assert!(!described.body.contains("未修正"));
17417        }
17418    }
17419
17420    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17421        let mut f = summary_fixture();
17422        if !review_only {
17423            f.runner.state.candidates[0].agent = "alpha".into();
17424        }
17425        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17426        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17427        fixed
17428            .fix
17429            .as_mut()
17430            .unwrap()
17431            .rejected
17432            .push(crate::verdict::Rejection {
17433                id: "R1-1-1".into(),
17434                why: "呼び出されないコードのため辞退".into(),
17435            });
17436        let mut clean = clean_round_with_minor_findings();
17437        clean.verified_head = Some(head.clone());
17438        f.runner.state.reviews = vec![fixed, clean];
17439        ok_gate(&mut f.runner.state, Some(&head));
17440        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17441        f.runner.state.config.agents.push(scripted("beta", reply));
17442        f.runner.state.config.roles.language_judge =
17443            Some(crate::config::AgentChoice::One("beta".into()));
17444        f
17445    }
17446
17447    #[tokio::test]
17448    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17449        for review_only in [false, true] {
17450            let mut f = rewrite_fixture_with_findings(review_only,
17451                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17452            let m = posted(&mut f).await;
17453            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17454            let expected =
17455                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17456            assert!(m.body.contains(&expected));
17457            assert!(m.body.contains("## Review rounds"));
17458            assert!(
17459                m.body
17460                    .contains("Gate (`verify.gate`): passed on the final commit")
17461            );
17462            assert!(
17463                m.body
17464                    .contains("Tests (`verify.e2e`): passed on the final commit")
17465            );
17466            let c = f.runner.state.change_summary.as_ref().unwrap();
17467            assert!(
17468                m.body
17469                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17470            );
17471            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17472            assert!(crate::pr_outcome::covers_required_findings(
17473                &f.runner.state,
17474                &m.body
17475            ));
17476            assert!(m.body.contains("documentation remains incomplete"));
17477            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17478            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17479            assert!(f.runner.state.github_text.is_none());
17480        }
17481    }
17482
17483    #[tokio::test]
17484    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17485        for review_only in [false, true] {
17486            for reply in [
17487                "TITLE: fix: public description\nA short English description.",
17488                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17489            ] {
17490                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17491                let winner = f.runner.state.candidates[0].clone();
17492                let posted = f
17493                    .runner
17494                    .guarded_pr_message(&winner, None, true)
17495                    .await
17496                    .unwrap();
17497                assert!(posted.is_none());
17498                assert!(f.runner.state.parked);
17499                let withheld = f.runner.state.github_text.as_ref().unwrap();
17500                assert!(
17501                    withheld.rewrite.is_none(),
17502                    "an incomplete rewrite must not be selectable"
17503                );
17504                let q = ask::Questions::open()
17505                    .get(withheld.question.as_ref().unwrap())
17506                    .unwrap();
17507                assert!(
17508                    !q.choices
17509                        .contains(&crate::github_text::USE_REWRITE.to_owned())
17510                );
17511                assert!(
17512                    q.detail.contains("R1-1-1")
17513                        && q.detail.contains("R2-1-1")
17514                        && q.detail.contains("R2-1-2")
17515                );
17516            }
17517        }
17518    }
17519
17520    #[tokio::test]
17521    async fn japanese_reports_stay_local_in_both_public_run_modes() {
17522        for review_only in [false, true] {
17523            let mut f = summary_fixture();
17524            if !review_only {
17525                f.runner.state.candidates[0].agent = "alpha".into();
17526            }
17527            f.runner.state.instruction = "内部タスクの説明".into();
17528            f.runner.state.candidates[0].summary = "内部の候補報告".into();
17529            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17530            f.runner.state.reviews = vec![
17531                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17532                round_with_notes(2, Some("二回目の内部修正報告です。")),
17533                passing_round(3, Some(&head)),
17534            ];
17535            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17536            ok_gate(&mut f.runner.state, Some(&head));
17537            let captured = f.repo.join("public-prompt.txt");
17538            let author = &mut f.runner.state.config.agents[0];
17539            author.command = vec![
17540                "sh".into(),
17541                "-c".into(),
17542                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17543            ];
17544            author.env.insert(
17545                "PROMPT_FILE".into(),
17546                captured.to_string_lossy().into_owned(),
17547            );
17548            let m = posted(&mut f).await;
17549            assert!(m.body.contains("Adds `own.txt`"));
17550            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17551            assert!(m.body.contains("R1-1-1"));
17552            assert!(
17553                m.body
17554                    .contains("Gate (`verify.gate`): passed on the final commit")
17555            );
17556            assert!(
17557                m.body
17558                    .contains("Tests (`verify.e2e`): passed on the final commit")
17559            );
17560            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17561            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17562            let prompt = std::fs::read_to_string(captured).unwrap();
17563            assert!(prompt.contains(&format!(
17564                "git diff {} {head}",
17565                f.runner.state.change_summary.as_ref().unwrap().base
17566            )));
17567            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17568            assert_eq!(
17569                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17570                "二回目の内部修正報告です。"
17571            );
17572        }
17573    }
17574
17575    #[tokio::test]
17576    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17577        for review_only in [false, true] {
17578            let mut f = summary_fixture();
17579            if !review_only {
17580                f.runner.state.candidates[0].agent = "alpha".into();
17581            }
17582            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17583            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17584            std::fs::write(&f.reply, "no usable description").unwrap();
17585            let failed = posted(&mut f).await;
17586            assert!(
17587                failed
17588                    .body
17589                    .contains("No description of what this change does could be generated")
17590            );
17591            assert!(failed.body.is_ascii());
17592            assert!(f.runner.state.github_text.is_none());
17593            assert!(f.runner.state.change_summary.is_none());
17594            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17595            let evidence = crate::pr_outcome::summary_evidence(
17596                &f.runner.state,
17597                crate::pr_outcome::FinalHead::At(&head),
17598            );
17599            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17600            assert!(
17601                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17602                    .exists()
17603            );
17604            f.runner.state =
17605                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17606            std::fs::write(
17607                &f.reply,
17608                "TITLE: feat(own): add the own file\nAdds the committed own file.",
17609            )
17610            .unwrap();
17611            let retried = posted(&mut f).await;
17612            assert!(retried.body.contains("Adds the committed own file."));
17613            assert_eq!(
17614                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17615                "日本語の修正報告"
17616            );
17617        }
17618    }
17619
17620    #[tokio::test]
17621    async fn same_head_verification_and_findings_invalidate_public_prose() {
17622        let mut f = summary_fixture();
17623        posted(&mut f).await;
17624        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17625        let original = f
17626            .runner
17627            .state
17628            .change_summary
17629            .as_ref()
17630            .unwrap()
17631            .evidence
17632            .clone();
17633        ok_gate(&mut f.runner.state, Some(&head));
17634        std::fs::write(
17635            &f.reply,
17636            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17637        )
17638        .unwrap();
17639        let verified = posted(&mut f).await;
17640        assert!(verified.body.contains("now verified"));
17641        assert_ne!(
17642            f.runner.state.change_summary.as_ref().unwrap().evidence,
17643            original
17644        );
17645        let mut round = round_with_notes(1, Some("内部報告"));
17646        round.reviews.push(ReviewRecord {
17647            attempts: 0,
17648            reviewer: 1,
17649            agent: "alpha".into(),
17650            summary: "内部レビュー".into(),
17651            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17652            vote: None,
17653            failed: None,
17654            duration_ms: 0,
17655        });
17656        round
17657            .fix
17658            .as_mut()
17659            .unwrap()
17660            .rejected
17661            .push(crate::verdict::Rejection {
17662                id: "R1-1-1".into(),
17663                why: "呼び出し元から到達できない".into(),
17664            });
17665        f.runner.state.reviews = vec![round];
17666        // A description omitting the required ID is unusable, even with a title.
17667        let omitted = posted(&mut f).await;
17668        assert!(omitted.body.contains("No description"));
17669        assert!(omitted.body.contains("R1-1-1"));
17670        assert!(!omitted.body.contains("到達"));
17671        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17672        let described = posted(&mut f).await;
17673        assert!(described.body.contains("unreachable from callers"));
17674        assert!(described.body.contains("R1-1-1"));
17675        assert!(!described.body.contains("到達"));
17676        let evidence = f
17677            .runner
17678            .state
17679            .change_summary
17680            .as_ref()
17681            .unwrap()
17682            .evidence
17683            .clone();
17684        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17685        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17686        assert!(posted(&mut f).await.body.contains("rationale updated"));
17687        assert_ne!(
17688            f.runner.state.change_summary.as_ref().unwrap().evidence,
17689            evidence
17690        );
17691        // Old serialized summaries have no evidence fingerprint and must regenerate.
17692        f.runner
17693            .state
17694            .change_summary
17695            .as_mut()
17696            .unwrap()
17697            .evidence
17698            .clear();
17699        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17700        assert!(posted(&mut f).await.body.contains("current specification"));
17701    }
17702
17703    #[tokio::test]
17704    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17705        let mut f = summary_fixture();
17706        let m = posted(&mut f).await;
17707        assert_eq!(m.title, "feat(own): add the own file");
17708        assert!(
17709            m.body
17710                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17711            "{}",
17712            m.body
17713        );
17714        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17715        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17716        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17717        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17718        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17719        let kept = f.runner.state.change_summary.as_ref().unwrap();
17720        assert_eq!(
17721            (kept.head.as_str(), kept.base.as_str()),
17722            (head.as_str(), base.as_str())
17723        );
17724        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17725    }
17726
17727    #[tokio::test]
17728    async fn review_only_summary_receives_the_configured_github_language() {
17729        let mut f = summary_fixture();
17730        f.runner.state.config.graph.github_language = "ja".into();
17731        let captured = f.repo.join("summary-prompt.txt");
17732        let author = &mut f.runner.state.config.agents[0];
17733        author.command = vec![
17734            "sh".into(),
17735            "-c".into(),
17736            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17737        ];
17738        author.env.insert(
17739            "PROMPT_FILE".into(),
17740            captured.to_string_lossy().into_owned(),
17741        );
17742        std::fs::write(
17743            &f.reply,
17744            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17745        )
17746        .unwrap();
17747        let m = posted(&mut f).await;
17748        let prompt = std::fs::read_to_string(captured).unwrap();
17749        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17750        assert!(!prompt.contains("in English"));
17751        assert_eq!(m.title, "feat(own): ファイルを追加");
17752        assert!(m.body.contains("own.txt を追加します。"));
17753    }
17754
17755    #[tokio::test]
17756    async fn review_fixes_come_after_the_whole_change_summary() {
17757        let mut f = summary_fixture();
17758        f.runner.state.reviews = vec![
17759            round_with_notes(1, Some("handled the PATH case")),
17760            round_with_notes(2, None),
17761        ];
17762        let m = posted(&mut f).await;
17763        let summary = m.body.find("## Summary").expect("summary");
17764        let fixes = m.body.find("## Review rounds").expect("review rounds");
17765        assert!(summary < fixes, "{}", m.body);
17766        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17767        assert!(m.body.contains("## Final verification"), "{}", m.body);
17768    }
17769
17770    #[tokio::test]
17771    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17772        let mut f = summary_fixture();
17773        let first = posted(&mut f).await;
17774        let first_head = f.runner.state.change_summary.clone().unwrap().head;
17775        // The same head reuses the record: a different reply is not read.
17776        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17777        let again = posted(&mut f).await;
17778        assert_eq!((again.title, again.body), (first.title, first.body));
17779
17780        let g = |args: &[&str]| {
17781            let out = std::process::Command::new("git")
17782                .args(args)
17783                .current_dir(&f.repo)
17784                .quiet()
17785                .output()
17786                .expect("spawn git");
17787            assert!(out.status.success(), "git {args:?}");
17788        };
17789        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17790        g(&["add", "-A"]);
17791        g(&["commit", "-q", "-m", "fix(own): handle more"]);
17792        let second = posted(&mut f).await;
17793        let kept = f.runner.state.change_summary.clone().unwrap();
17794        assert_ne!(kept.head, first_head);
17795        assert!(second.body.contains("Not read yet."), "{}", second.body);
17796        assert!(!second.body.contains("so the first summary is read"));
17797        assert_eq!(second.title, "feat(own): other");
17798    }
17799
17800    #[tokio::test]
17801    async fn an_unavailable_summary_is_stated_and_never_invented() {
17802        let mut f = summary_fixture();
17803        std::fs::write(&f.reply, "no title line here").unwrap();
17804        let m = posted(&mut f).await;
17805        assert!(
17806            m.body
17807                .contains("No description of what this change does could be generated"),
17808            "{}",
17809            m.body
17810        );
17811        assert!(m.body.contains("internal reports remain in local records"));
17812        assert!(f.runner.state.change_summary.is_none());
17813        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17814        // A preview asks nobody and claims nothing.
17815        let winner = f.runner.state.candidates[0].clone();
17816        let preview = f
17817            .runner
17818            .guarded_pr_message(&winner, None, false)
17819            .await
17820            .unwrap()
17821            .unwrap();
17822        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17823    }
17824
17825    #[test]
17826    fn pr_message_review_single_commit_uses_its_subject() {
17827        let state = review_state(&["feat(nats): per-role user"]);
17828        let m = pr_message(&state, 'A');
17829        assert_eq!(m.title, "feat(nats): per-role user");
17830        assert!(!m.body.contains("Review the work already"), "{}", m.body);
17831        assert!(m.body.contains("## Commits under review"), "{}", m.body);
17832    }
17833
17834    #[test]
17835    fn pr_message_review_multi_commit_takes_the_oldest() {
17836        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17837        let m = pr_message(&state, 'A');
17838        assert_eq!(m.title, "feat: the change");
17839        for s in ["feat: the change", "fix: typo", "fix: again"] {
17840            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17841        }
17842    }
17843
17844    #[test]
17845    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17846        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17847            let mut state = review_state(&[first, "fix: later fixup"]);
17848            state.candidates[0].branch = "機能/ログイン".to_owned();
17849            let m = pr_message(&state, 'A');
17850            assert!(
17851                m.title.starts_with("chore: land candidate A of run"),
17852                "{}",
17853                m.title
17854            );
17855        }
17856    }
17857
17858    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17859        BranchFacts {
17860            commits: commits
17861                .iter()
17862                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17863                .collect(),
17864            stat: stat.to_owned(),
17865        }
17866    }
17867
17868    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17869        let mut r = review_round(true, 0, 1, 1, true, true);
17870        r.round = round;
17871        r.fix = notes.map(|n| FixRecord {
17872            agent: "fixer".to_owned(),
17873            addressed: Vec::new(),
17874            rejected: Vec::new(),
17875            notes: n.to_owned(),
17876            committed: true,
17877            failed: None,
17878            duration_ms: 0,
17879            continuation: None,
17880        });
17881        r
17882    }
17883
17884    #[test]
17885    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17886        let state = review_state(&["ignored"]);
17887        let f = facts(
17888            &[
17889                (
17890                    "fix(login): resolve PATH on macOS",
17891                    "Login shells skip rc files.",
17892                ),
17893                ("fix: address review", ""),
17894            ],
17895            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17896        );
17897        let m = pr_message_with(&state, 'A', Some(&f));
17898        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17899        assert!(
17900            m.body
17901                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
17902            "{}",
17903            m.body
17904        );
17905        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17906        assert!(m.body.contains("## Diff stat"), "{}", m.body);
17907        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17908        for banned in [
17909            "Review the work already",
17910            "no task statement",
17911            "Original task",
17912        ] {
17913            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17914        }
17915        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17916    }
17917
17918    #[test]
17919    fn pr_message_review_truncates_a_huge_first_commit_body() {
17920        let state = review_state(&["ignored"]);
17921        let f = facts(
17922            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17923            "s",
17924        );
17925        let m = pr_message_with(&state, 'A', Some(&f));
17926        assert!(m.body.len() < 30_000, "{}", m.body.len());
17927        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17928        assert!(
17929            m.body.contains("- ... 1 more commit(s)"),
17930            "{}",
17931            m.body.len()
17932        );
17933        assert!(m.body.ends_with("magi:candidate-a\n"));
17934    }
17935
17936    #[test]
17937    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17938        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17939        assert_eq!(m.title, "feat: x");
17940        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17941        assert!(!m.body.contains("Diff stat"), "{}", m.body);
17942        assert!(!m.body.contains("no task statement"), "{}", m.body);
17943    }
17944
17945    #[test]
17946    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17947        let mut state = review_state(&["日本語の件名"]);
17948        state.candidates[0].branch = "fix/macos-login-path".to_owned();
17949        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17950        state.candidates[0].branch = "機能/ログイン".to_owned();
17951        assert!(
17952            pr_message(&state, 'A')
17953                .title
17954                .starts_with("chore: land candidate A")
17955        );
17956    }
17957
17958    #[test]
17959    fn pr_message_review_fixes_survive_a_clean_final_round() {
17960        let mut state = review_state(&["feat: x"]);
17961        state.reviews = vec![
17962            round_with_notes(1, Some("handled the PATH case")),
17963            round_with_notes(2, None),
17964        ];
17965        let body = pr_message(&state, 'A').body;
17966        assert!(
17967            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17968            "{body}"
17969        );
17970        assert!(body.contains("### Round 1"), "{body}");
17971
17972        state.reviews = vec![
17973            round_with_notes(1, Some("first fix")),
17974            round_with_notes(2, Some("")),
17975            round_with_notes(3, Some("second fix")),
17976            round_with_notes(4, None),
17977        ];
17978        let body = pr_message(&state, 'A').body;
17979        assert!(
17980            body.contains("### Round 1") && !body.contains("first fix"),
17981            "{body}"
17982        );
17983        assert!(
17984            body.contains("### Round 3") && !body.contains("second fix"),
17985            "{body}"
17986        );
17987        assert!(body.contains("### Round 2"), "{body}");
17988    }
17989
17990    const FINAL: &str = "abcdef1234567890";
17991
17992    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17993        let mut r = round_with_notes(round, None);
17994        r.verified_head = verified.map(str::to_owned);
17995        r
17996    }
17997
17998    fn ok_gate(state: &mut RunState, head: Option<&str>) {
17999        state.gate = vec![CommandOutcome {
18000            command: "gate".to_owned(),
18001            code: Some(0),
18002            output_tail: String::new(),
18003            duration_ms: 0,
18004            resource_blocked: false,
18005        }];
18006        state.gate_ran = true;
18007        state.gate_head = head.map(str::to_owned);
18008    }
18009
18010    fn body_at(state: &RunState) -> String {
18011        pr_message_raw(
18012            state,
18013            'A',
18014            None,
18015            crate::pr_outcome::FinalHead::At(FINAL),
18016            ChangeNote::NotAsked,
18017        )
18018        .body
18019    }
18020
18021    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
18022
18023    #[test]
18024    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
18025        let mut state = review_state(&["feat: x"]);
18026        state.reviews = vec![
18027            round_with_notes(1, Some(SANDBOXED)),
18028            passing_round(2, Some(FINAL)),
18029        ];
18030        ok_gate(&mut state, Some(FINAL));
18031        let body = body_at(&state);
18032        assert!(!body.contains("Handled the PATH case."), "{body}");
18033        assert!(!body.contains("could not commit"), "{body}");
18034        assert!(!body.contains("could not run"), "{body}");
18035        assert!(
18036            body.contains("The fix is committed on the branch"),
18037            "{body}"
18038        );
18039        assert!(!body.contains("sandbox"), "{body}");
18040        assert!(
18041            body.contains("Tests (`verify.e2e`): passed on the final commit"),
18042            "{body}"
18043        );
18044        assert!(
18045            body.contains("Gate (`verify.gate`): passed on the final commit"),
18046            "{body}"
18047        );
18048        // The raw report is untouched in the record.
18049        assert!(
18050            state.reviews[0]
18051                .fix
18052                .as_ref()
18053                .unwrap()
18054                .notes
18055                .contains("could not commit")
18056        );
18057    }
18058
18059    #[test]
18060    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
18061        let mut state = review_state(&["feat: x"]);
18062        let mut r1 = round_with_notes(1, Some(SANDBOXED));
18063        r1.fix.as_mut().unwrap().committed = false;
18064        let mut r2 = passing_round(2, Some(FINAL));
18065        r2.e2e[0].code = Some(1);
18066        state.reviews = vec![r1, r2];
18067        let body = body_at(&state);
18068        assert!(!body.contains("could not commit"), "{body}");
18069        assert!(!body.contains("could not run"), "{body}");
18070        assert!(body.contains("This round produced no commit"), "{body}");
18071        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
18072        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
18073        assert!(!body.contains("passed"), "{body}");
18074    }
18075
18076    #[test]
18077    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
18078        let mut state = review_state(&["feat: x"]);
18079        state.reviews = vec![
18080            round_with_notes(1, Some(SANDBOXED)),
18081            passing_round(2, Some("0123456789abcdef")),
18082        ];
18083        ok_gate(&mut state, Some("0123456789abcdef"));
18084        let body = body_at(&state);
18085        assert!(!body.contains("HTTP tests could not run"), "{body}");
18086        // The commit claim is still settled by the structured record.
18087        assert!(!body.contains("could not commit"), "{body}");
18088        assert!(
18089            body.contains("passed on an earlier commit (`0123456`)"),
18090            "{body}"
18091        );
18092        assert!(!body.contains("passed on the final commit"), "{body}");
18093
18094        // A gate recorded before `gate_head` existed is "not recorded".
18095        ok_gate(&mut state, None);
18096        let body = body_at(&state);
18097        assert!(
18098            body.contains("passed on a commit that was not recorded"),
18099            "{body}"
18100        );
18101    }
18102
18103    #[test]
18104    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
18105        let state = state_with_summary(
18106            "add retries to the client",
18107            "TITLE: feat: retries\n\nDid it.",
18108        );
18109        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
18110        assert_eq!(m.title, "feat: retries");
18111        assert!(
18112            m.body.contains("<summary>Original task</summary>"),
18113            "{}",
18114            m.body
18115        );
18116        assert!(!m.body.contains("Commits under review"), "{}", m.body);
18117        assert!(
18118            m.body
18119                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
18120            "{}",
18121            m.body
18122        );
18123    }
18124
18125    #[test]
18126    fn pr_message_review_bounds_a_long_english_subject() {
18127        let long = format!("feat: {}", "word ".repeat(100));
18128        let m = pr_message(&review_state(&[&long]), 'A');
18129        assert!(m.title.starts_with("feat: word"), "{}", m.title);
18130        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18131    }
18132
18133    #[test]
18134    fn pr_message_implementation_run_is_unchanged_by_review_support() {
18135        let state = state_with_summary("add retries\n\ndetails", "- did some things");
18136        let m = pr_message(&state, 'A');
18137        assert_eq!(m.title, "add retries");
18138        assert!(m.body.contains("<summary>Original task</summary>"));
18139        assert!(!m.body.contains("Commits under review"));
18140        assert_eq!(landing_subject_source(&state), state.instruction);
18141    }
18142
18143    #[test]
18144    fn review_run_squash_subject_is_the_change_not_the_prompt() {
18145        let state = review_state(&["feat: the change", "fix: typo"]);
18146        let source = landing_subject_source(&state);
18147        assert_eq!(land::merge_subject("", &source), "feat: the change");
18148        assert_eq!(
18149            land::merge_subject("magi: candidate A (uncommitted work)", &source),
18150            "feat: the change"
18151        );
18152        // An operator's rename still wins.
18153        assert_eq!(
18154            land::merge_subject("feat: renamed by hand", &source),
18155            "feat: renamed by hand"
18156        );
18157        let blank = review_state(&["日本語"]);
18158        assert!(
18159            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
18160        );
18161    }
18162
18163    #[test]
18164    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
18165        let state = review_state(&["feat: the change"]);
18166        let source = landing_subject_source(&state);
18167        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
18168        assert_eq!(
18169            land::merge_subject(landing_title(&state, old), &source),
18170            "feat: the change"
18171        );
18172        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
18173        let task = state_with_summary("add retries", "");
18174        assert_eq!(landing_title(&task, old), old);
18175    }
18176
18177    #[test]
18178    fn pr_message_describes_the_change_not_the_task() {
18179        let state = state_with_summary(
18180            "今回やってほしいこと: results projector を直す",
18181            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
18182        );
18183        let m = pr_message(&state, 'A');
18184        assert_eq!(m.title, "fix(web): batch the runs list reads");
18185        assert!(
18186            m.body.starts_with("## Summary\n\n- reads run.json once"),
18187            "{}",
18188            m.body
18189        );
18190        assert!(!m.body.contains("TITLE:"), "{}", m.body);
18191        let task_at = m.body.find("今回やってほしいこと").unwrap();
18192        let details_at = m.body.find("<details>").unwrap();
18193        assert!(
18194            details_at < task_at,
18195            "the task lives inside <details>: {}",
18196            m.body
18197        );
18198        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
18199        assert!(m.body.contains("magi:candidate-a"));
18200    }
18201
18202    #[test]
18203    fn pr_message_falls_back_to_the_task_without_a_title_line() {
18204        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
18205        let m = pr_message(&state, 'A');
18206        assert_eq!(m.title, "add retries");
18207        assert!(
18208            m.body.contains("## Summary\n\n- did some things"),
18209            "{}",
18210            m.body
18211        );
18212
18213        let none = RunState::new(
18214            PathBuf::from("/repo"),
18215            "main".to_owned(),
18216            "abc1234".to_owned(),
18217            "add retries".to_owned(),
18218            Config::default(),
18219        );
18220        let m = pr_message(&none, 'A');
18221        assert_eq!(m.title, "add retries");
18222        assert!(!m.body.contains("## Summary"), "{}", m.body);
18223    }
18224
18225    #[test]
18226    fn pr_message_refuses_the_candidate_commit_subject() {
18227        for bad in [
18228            "TITLE: magi: candidate A (uncommitted work)",
18229            "TITLE: chore: stuff (uncommitted work)",
18230            "TITLE:   ",
18231        ] {
18232            let state = state_with_summary("add retries", bad);
18233            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
18234        }
18235    }
18236
18237    #[test]
18238    fn pr_message_bounds_a_very_long_task_and_title() {
18239        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
18240        let state = state_with_summary(&long, "- nothing");
18241        let m = pr_message(&state, 'A');
18242        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18243        assert!(!m.title.contains('\n'));
18244
18245        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
18246        let m = pr_message(&state, 'A');
18247        assert!(m.title.starts_with("feat: "));
18248        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18249        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
18250    }
18251
18252    fn long_title_of(instruction: &str) -> String {
18253        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
18254    }
18255
18256    #[test]
18257    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
18258        let first = "Make the landing path keep a readable title for long tasks";
18259        let line = format!(
18260            "{first}. {}",
18261            "And then keep going with more words ".repeat(20)
18262        );
18263        let t = long_title_of(&line);
18264        assert_eq!(t, first);
18265        assert!(!t.starts_with("chore: land"));
18266    }
18267
18268    #[test]
18269    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
18270        let line = "word ".repeat(200);
18271        let t = long_title_of(&line);
18272        assert!(t.ends_with("word..."), "{t}");
18273        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
18274    }
18275
18276    #[test]
18277    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
18278        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
18279            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
18280        }
18281    }
18282
18283    #[test]
18284    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
18285        let mut state = state_with_summary("再試行を追加する", "- nothing");
18286        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
18287        state.config.graph.github_language = "ja".into();
18288        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
18289    }
18290
18291    #[test]
18292    fn pr_message_title_limit_is_exact() {
18293        let at = "a".repeat(PR_TITLE_MAX);
18294        assert_eq!(long_title_of(&at), at);
18295        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
18296        assert!(over.ends_with("..."), "{over}");
18297        assert_eq!(over.chars().count(), PR_TITLE_MAX);
18298    }
18299
18300    #[test]
18301    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
18302        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
18303        let t = long_title_of(&line);
18304        assert!(t.ends_with("..."), "{t}");
18305        assert!(t.is_ascii(), "{t}");
18306    }
18307
18308    #[test]
18309    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
18310        let line = format!(
18311            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
18312            "plus more filler words ".repeat(20)
18313        );
18314        let t = long_title_of(&line);
18315        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
18316    }
18317
18318    #[test]
18319    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
18320        let line = format!("\n\n# {}", "title words ".repeat(40));
18321        let state = state_with_summary(&line, "- nothing");
18322        let m = pr_message(&state, 'A');
18323        assert!(m.title.starts_with("title words"), "{}", m.title);
18324        assert_eq!(
18325            land::merge_subject(&m.title, &landing_subject_source(&state)),
18326            m.title
18327        );
18328        // An operator's rename wins untouched.
18329        assert_eq!(
18330            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
18331            "feat: renamed by hand"
18332        );
18333    }
18334
18335    #[tokio::test]
18336    async fn github_text_rewrite_is_bounded_and_falls_back() {
18337        crate::run::pin_test_home();
18338        for (reply, accepted) in [
18339            (
18340                "TITLE: fix: retries\nAdd retries for failed requests.",
18341                true,
18342            ),
18343            (
18344                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18345                false,
18346            ),
18347            (
18348                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18349                true,
18350            ),
18351            (
18352                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18353                true,
18354            ),
18355            (
18356                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18357                true,
18358            ),
18359            (
18360                "TITLE: fix: retries\n日本語の説明です。 token=secret",
18361                false,
18362            ),
18363            ("TITLE: fix: retries\n", false),
18364        ] {
18365            let dir = tempfile::tempdir().unwrap();
18366            let mut runner = runner_at(RunStatus::Gating);
18367            runner.state = state_with_summary(
18368                "add retries",
18369                "TITLE: fix: retries\n日本語の説明を書きます。",
18370            );
18371            runner.state.repo = dir.path().to_owned();
18372            runner.state.candidates[0].worktree = dir.path().to_owned();
18373            let mut author = spec("alpha");
18374            author.command = vec![
18375                "sh".into(),
18376                "-c".into(),
18377                "printf '%s' \"$REWRITE_REPLY\"".into(),
18378            ];
18379            author.env.insert("REWRITE_REPLY".into(), reply.into());
18380            runner.state.config.agents = vec![author];
18381            seed_public_summary(&mut runner);
18382            let winner = runner.state.candidates[0].clone();
18383            let first = runner
18384                .guarded_pr_message(&winner, None, true)
18385                .await
18386                .unwrap();
18387            let message = if accepted {
18388                first.expect("a passing rewrite never asks")
18389            } else {
18390                // Withheld: the run parks on a question and posts nothing yet.
18391                assert!(first.is_none());
18392                assert!(runner.state.parked);
18393                let store = ask::Questions::open();
18394                let asked: Vec<_> = store
18395                    .list()
18396                    .into_iter()
18397                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18398                    .collect();
18399                assert_eq!(asked.len(), 1);
18400                assert!(!asked[0].detail.contains("secret"));
18401                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18402                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18403                assert_eq!(
18404                    asked[0]
18405                        .choices
18406                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
18407                    rewrite.is_some()
18408                );
18409                // Silence: the question lapses and the neutral text is used,
18410                // without a second rewrite call or a second question.
18411                store
18412                    .update(&asked[0].id, |q| {
18413                        q.abandon("test");
18414                        Ok(())
18415                    })
18416                    .unwrap();
18417                let again = runner
18418                    .guarded_pr_message(&winner, None, true)
18419                    .await
18420                    .unwrap()
18421                    .expect("an abandoned question falls back");
18422                let n = store
18423                    .list()
18424                    .into_iter()
18425                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18426                    .count();
18427                assert_eq!(n, 1, "asked once per rejected text");
18428                again
18429            };
18430            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18431            assert!(!message.body.contains("secret"));
18432            assert!(!message.body.contains("prefix"));
18433            assert!(!message.title.contains("secret"));
18434            assert_eq!(
18435                message.body.contains("Add retries for failed requests."),
18436                accepted
18437            );
18438            assert_eq!(
18439                runner.state.seats["github-text-rewrite"].turns, 1,
18440                "only one rewrite invocation"
18441            );
18442            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18443            if accepted {
18444                assert!(runner.state.github_text.is_none());
18445                assert!(!runner.state.parked);
18446                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18447                assert!(message.body.contains("## Final verification"));
18448                assert!(message.body.contains("Evidence: final diff"));
18449                assert!(
18450                    message
18451                        .body
18452                        .contains(&format!("magi:run/{}", runner.state.id))
18453                );
18454            }
18455            if !accepted {
18456                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18457                assert!(
18458                    message
18459                        .body
18460                        .contains(&format!("magi:run/{}", runner.state.id))
18461                );
18462            }
18463        }
18464    }
18465
18466    fn scripted(id: &str, reply: &str) -> AgentSpec {
18467        let mut a = spec(id);
18468        a.command = vec![
18469            "sh".into(),
18470            "-c".into(),
18471            "printf '%s' \"$REWRITE_REPLY\"".into(),
18472        ];
18473        a.env.insert("REWRITE_REPLY".into(), reply.into());
18474        a
18475    }
18476
18477    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18478    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18479
18480    /// Gate tests inject generated public output, not internal reports.
18481    fn seed_public_summary(runner: &mut Runner) {
18482        let repo = &runner.state.repo;
18483        let g = |args: &[&str]| {
18484            let out = std::process::Command::new("git")
18485                .args(args)
18486                .current_dir(repo)
18487                .quiet()
18488                .output()
18489                .unwrap();
18490            assert!(out.status.success(), "git {args:?}");
18491            String::from_utf8_lossy(&out.stdout).trim().to_owned()
18492        };
18493        g(&["init", "-q", "-b", "main"]);
18494        g(&[
18495            "-c",
18496            "user.name=Test",
18497            "-c",
18498            "user.email=test@example.invalid",
18499            "commit",
18500            "-q",
18501            "--allow-empty",
18502            "-m",
18503            "feat: fixture",
18504        ]);
18505        let head = g(&["rev-parse", "HEAD"]);
18506        g(&["update-ref", "refs/remotes/origin/main", &head]);
18507        let evidence = crate::pr_outcome::summary_evidence(
18508            &runner.state,
18509            crate::pr_outcome::FinalHead::At(&head),
18510        );
18511        let generated = &runner.state.candidates[0].summary;
18512        runner.state.change_summary = Some(crate::run::ChangeSummary {
18513            base: head.clone(),
18514            head,
18515            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18516            title: summary_title(generated).unwrap(),
18517            body: summary_without_title(generated),
18518        });
18519    }
18520
18521    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18522        let mut runner = runner_at(RunStatus::Gating);
18523        runner.state = state_with_summary("add retries", JA_SUMMARY);
18524        if review_only {
18525            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18526        }
18527        runner.state.repo = dir.to_owned();
18528        runner.state.candidates[0].worktree = dir.to_owned();
18529        runner.state.config.agents = agents;
18530        seed_public_summary(&mut runner);
18531        runner
18532    }
18533
18534    fn gate_questions(runner: &Runner) -> usize {
18535        ask::Questions::open()
18536            .list()
18537            .into_iter()
18538            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18539            .count()
18540    }
18541
18542    #[tokio::test]
18543    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18544        crate::run::pin_test_home();
18545        let dir = tempfile::tempdir().unwrap();
18546        let mut runner = rewrite_runner(
18547            dir.path(),
18548            true,
18549            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18550        );
18551        runner.state.config.roles.language_judge =
18552            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18553        let winner = runner.state.candidates[0].clone();
18554        let msg = runner
18555            .guarded_pr_message(&winner, None, true)
18556            .await
18557            .unwrap()
18558            .expect("an English rewrite posts without asking");
18559        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18560        assert!(msg.body.contains("<details>kept</details>"));
18561        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18562        assert_eq!(gate_questions(&runner), 0);
18563        assert!(runner.state.github_text.is_none());
18564        assert!(!runner.state.seats.contains_key("impl-A"));
18565        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18566    }
18567
18568    #[tokio::test]
18569    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18570        crate::run::pin_test_home();
18571        let dir = tempfile::tempdir().unwrap();
18572        let mut runner = rewrite_runner(
18573            dir.path(),
18574            false,
18575            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18576        );
18577        runner.state.config.roles.language_judge =
18578            Some(crate::config::AgentChoice::One("beta".into()));
18579        runner.state.seats.insert(
18580            "impl-A".into(),
18581            SeatState::new("impl-A", "alpha", runner.state.seed),
18582        );
18583        let winner = runner.state.candidates[0].clone();
18584        let msg = runner
18585            .guarded_pr_message(&winner, None, true)
18586            .await
18587            .unwrap()
18588            .expect("rewritten");
18589        assert!(msg.body.contains("<details>kept</details>"));
18590        assert_eq!(runner.state.seats["impl-A"].turns, 0);
18591        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18592        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18593    }
18594
18595    #[tokio::test]
18596    async fn rewrite_reuses_a_resumable_implementer_session() {
18597        crate::run::pin_test_home();
18598        let dir = tempfile::tempdir().unwrap();
18599        let mut runner = rewrite_runner(
18600            dir.path(),
18601            false,
18602            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18603        );
18604        runner.state.config.roles.language_judge =
18605            Some(crate::config::AgentChoice::One("beta".into()));
18606        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18607        seat.turns = 1;
18608        runner.state.seats.insert("impl-A".into(), seat);
18609        let winner = runner.state.candidates[0].clone();
18610        runner
18611            .guarded_pr_message(&winner, None, true)
18612            .await
18613            .unwrap()
18614            .expect("rewritten");
18615        assert_eq!(runner.state.seats["impl-A"].turns, 2);
18616        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18617    }
18618
18619    #[tokio::test]
18620    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
18621        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
18622        // session, was out of quota, and nothing else was asked.
18623        crate::run::pin_test_home();
18624        let dir = tempfile::tempdir().unwrap();
18625        let mut exhausted = spec("alpha");
18626        exhausted.command = vec!["false".into()];
18627        let mut runner = rewrite_runner(
18628            dir.path(),
18629            false,
18630            vec![exhausted, scripted("beta", EN_REWRITE)],
18631        );
18632        // The implementer's own agent is in the role too: it is not asked twice.
18633        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18634            "alpha".into(),
18635            "beta".into(),
18636        ]));
18637        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18638        seat.turns = 1;
18639        runner.state.seats.insert("impl-A".into(), seat);
18640        let winner = runner.state.candidates[0].clone();
18641        let msg = runner
18642            .guarded_pr_message(&winner, None, true)
18643            .await
18644            .unwrap()
18645            .expect("the role rewrites after the implementer fails");
18646        assert!(msg.body.contains("`src/net.rs`"));
18647        assert_eq!(
18648            runner.state.seats["impl-A"].turns, 2,
18649            "the implementer was tried once"
18650        );
18651        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18652        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18653    }
18654
18655    #[tokio::test]
18656    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18657        crate::run::pin_test_home();
18658        let dir = tempfile::tempdir().unwrap();
18659        let mut broken = spec("bad");
18660        broken.command = vec!["false".into()];
18661        let mut runner = rewrite_runner(
18662            dir.path(),
18663            true,
18664            vec![
18665                broken,
18666                scripted("noformat", "Add retries."),
18667                scripted("good", EN_REWRITE),
18668            ],
18669        );
18670        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18671            "bad".into(),
18672            "noformat".into(),
18673            "good".into(),
18674        ]));
18675        let winner = runner.state.candidates[0].clone();
18676        let msg = runner
18677            .guarded_pr_message(&winner, None, true)
18678            .await
18679            .unwrap()
18680            .expect("the third agent rewrites");
18681        assert!(msg.body.contains("`src/net.rs`"));
18682        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18683        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18684    }
18685
18686    #[tokio::test]
18687    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18688        crate::run::pin_test_home();
18689        // (agents, role): still Japanese; no TITLE; nothing resolvable.
18690        for (agents, role, saved) in [
18691            (
18692                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18693                Some("beta"),
18694                true,
18695            ),
18696            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18697            (Vec::new(), None, false),
18698        ] {
18699            let dir = tempfile::tempdir().unwrap();
18700            let mut runner = rewrite_runner(dir.path(), true, agents);
18701            runner.state.config.roles.language_judge =
18702                role.map(|r| crate::config::AgentChoice::One(r.into()));
18703            let winner = runner.state.candidates[0].clone();
18704            let first = runner
18705                .guarded_pr_message(&winner, None, true)
18706                .await
18707                .unwrap();
18708            assert!(first.is_none(), "withheld text parks the run");
18709            assert!(runner.state.parked);
18710            assert_eq!(gate_questions(&runner), 1);
18711            let g = runner.state.github_text.as_ref().unwrap();
18712            assert_eq!(g.rewrite.is_some(), saved);
18713            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18714        }
18715    }
18716
18717    #[tokio::test]
18718    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18719        crate::run::pin_test_home();
18720        let dir = tempfile::tempdir().unwrap();
18721        let mut runner = runner_at(RunStatus::Gating);
18722        runner.state =
18723            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18724        runner.state.repo = dir.path().to_owned();
18725        runner.state.candidates[0].worktree = dir.path().to_owned();
18726        let mut author = spec("alpha");
18727        author.command = vec![
18728            "sh".into(),
18729            "-c".into(),
18730            "printf '%s' \"$REWRITE_REPLY\"".into(),
18731        ];
18732        author.env.insert(
18733            "REWRITE_REPLY".into(),
18734            format!(
18735                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18736                "English sentence. ".repeat(600)
18737            ),
18738        );
18739        runner.state.config.agents = vec![author];
18740        seed_public_summary(&mut runner);
18741        let winner = runner.state.candidates[0].clone();
18742        assert!(
18743            runner
18744                .guarded_pr_message(&winner, None, true)
18745                .await
18746                .unwrap()
18747                .is_none()
18748        );
18749        let record = runner.state.github_text.clone().unwrap();
18750        let rewrite = record.rewrite.clone().unwrap();
18751        assert_eq!(rewrite.failed, ["body-language"]);
18752        assert!(
18753            rewrite
18754                .body
18755                .ends_with(&format!("magi:run/{}", runner.state.id))
18756        );
18757        let store = ask::Questions::open();
18758        let q = store.get(record.question.as_ref().unwrap()).unwrap();
18759        assert!(
18760            q.choices
18761                .contains(&crate::github_text::USE_REWRITE.to_owned())
18762        );
18763        for label in [
18764            "ORIGINAL",
18765            "FALLBACK",
18766            "REWRITE",
18767            "Rules still failed: body-language",
18768            "truncated",
18769        ] {
18770            assert!(q.detail.contains(label), "{label}");
18771        }
18772        let artifact = std::fs::read_to_string(crate::run::artifact_path(
18773            &runner.state,
18774            &crate::github_text::artifact_name(&record.fingerprint),
18775        ))
18776        .unwrap();
18777        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18778        // Restart from serialized state, then retry after a bad owner title.
18779        runner.state =
18780            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18781        store
18782            .update(&q.id, |q| {
18783                q.say("まだ日本語")?;
18784                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18785            })
18786            .unwrap();
18787        assert!(
18788            runner
18789                .guarded_pr_message(&winner, None, true)
18790                .await
18791                .unwrap()
18792                .is_none()
18793        );
18794        let retry = runner.state.github_text.as_ref().unwrap();
18795        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18796        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18797        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18798        assert!(
18799            q.posting_comparison
18800                .as_ref()
18801                .unwrap()
18802                .matches_detail(&q.detail)
18803        );
18804        store
18805            .update(&q.id, |q| {
18806                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18807            })
18808            .unwrap();
18809        let message = runner
18810            .guarded_pr_message(&winner, None, true)
18811            .await
18812            .unwrap()
18813            .unwrap();
18814        assert_eq!(message.title, rewrite.title);
18815        assert_eq!(message.body, rewrite.body);
18816        assert_eq!(
18817            runner.state.github_text.as_ref().unwrap().choice,
18818            Some(GithubTextChoice::Rewrite)
18819        );
18820        let again = runner
18821            .guarded_pr_message(&winner, None, true)
18822            .await
18823            .unwrap()
18824            .unwrap();
18825        assert_eq!(again.title, message.title);
18826        assert_eq!(again.body, message.body);
18827        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18828        assert!(
18829            !runner
18830                .state
18831                .events
18832                .iter()
18833                .any(|e| e.message.contains(&rewrite.title)
18834                    || e.message.contains("English sentence."))
18835        );
18836    }
18837
18838    #[tokio::test]
18839    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18840        crate::run::pin_test_home();
18841        let dir = tempfile::tempdir().unwrap();
18842        let mut runner = runner_at(RunStatus::Gating);
18843        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18844        runner.state.repo = dir.path().to_owned();
18845        runner.state.candidates[0].worktree = dir.path().to_owned();
18846        seed_public_summary(&mut runner);
18847        let winner = runner.state.candidates[0].clone();
18848        assert!(
18849            runner
18850                .guarded_pr_message(&winner, None, true)
18851                .await
18852                .unwrap()
18853                .is_none()
18854        );
18855        let record = runner.state.github_text.as_ref().unwrap();
18856        assert!(record.rewrite.is_none());
18857        // Records written before the additive field deserialize with no rewrite.
18858        let mut old = serde_json::to_value(record).unwrap();
18859        old.as_object_mut().unwrap().remove("rewrite");
18860        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18861        assert!(old.rewrite.is_none());
18862        let id = record.question.clone().unwrap();
18863        ask::Questions::open()
18864            .update(&id, |q| {
18865                q.choices.push(crate::github_text::USE_REWRITE.into());
18866                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18867            })
18868            .unwrap();
18869        let message = runner
18870            .guarded_pr_message(&winner, None, true)
18871            .await
18872            .unwrap()
18873            .unwrap();
18874        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18875        assert_eq!(
18876            runner.state.github_text.as_ref().unwrap().choice,
18877            Some(GithubTextChoice::Fallback)
18878        );
18879    }
18880
18881    #[tokio::test]
18882    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18883        crate::run::pin_test_home();
18884        let dir = tempfile::tempdir().unwrap();
18885        let mut runner = runner_at(RunStatus::Gating);
18886        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18887        runner.state.repo = dir.path().to_owned();
18888        runner.state.candidates[0].worktree = dir.path().to_owned();
18889        seed_public_summary(&mut runner);
18890        let winner = runner.state.candidates[0].clone();
18891        assert!(
18892            runner
18893                .guarded_pr_message(&winner, None, true)
18894                .await
18895                .unwrap()
18896                .is_none()
18897        );
18898        let store = ask::Questions::open();
18899        let find = |run: &str| {
18900            store
18901                .list()
18902                .into_iter()
18903                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18904                .unwrap()
18905        };
18906        let run = runner.state.id.clone();
18907        let q = find(&run);
18908        let fp = runner
18909            .state
18910            .github_text
18911            .as_ref()
18912            .unwrap()
18913            .fingerprint
18914            .clone();
18915        let name = crate::github_text::artifact_name(&fp);
18916        let path = crate::run::artifact_path(&runner.state, &name);
18917        assert!(q.detail.contains(&path.display().to_string()));
18918        assert!(q.detail.contains("title-language: title"));
18919        let saved = std::fs::read_to_string(&path).expect("artifact written");
18920        assert!(saved.contains("日本語のタイトル"));
18921        assert!(q.cwd.is_none());
18922        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18923        // A bad replacement is refused and asked about once more.
18924        let say = |id: &str, text: &str, choice: &str| {
18925            store
18926                .update(id, |q| {
18927                    q.thread.push(ask::Turn {
18928                        who: ask::Who::Operator,
18929                        body: text.to_owned(),
18930                        at: jiff::Timestamp::now(),
18931                        note: None,
18932                    });
18933                    q.answer(ask::Answer::Choice(choice.to_owned()))
18934                })
18935                .unwrap();
18936        };
18937        say(&q.id, "まだ日本語", "use my text");
18938        assert!(
18939            runner
18940                .guarded_pr_message(&winner, None, true)
18941                .await
18942                .unwrap()
18943                .is_none()
18944        );
18945        let second = find(&run);
18946        assert_ne!(second.id, q.id);
18947        assert!(second.detail.contains("did not pass"));
18948        // The retry still compares the original text; the rejected
18949        // replacement is named by category only.
18950        assert!(second.detail.contains("日本語のタイトル"));
18951        assert!(!second.detail.contains("まだ日本語"));
18952        assert!(second.detail.contains(&path.display().to_string()));
18953        say(&second.id, "fix: retry failed requests", "use my text");
18954        let message = runner
18955            .guarded_pr_message(&winner, None, true)
18956            .await
18957            .unwrap()
18958            .expect("a vetted replacement is posted");
18959        assert_eq!(message.title, "fix: retry failed requests");
18960        assert_eq!(
18961            runner
18962                .state
18963                .github_text
18964                .as_ref()
18965                .unwrap()
18966                .chosen_title
18967                .as_deref(),
18968            Some("fix: retry failed requests")
18969        );
18970        assert!(runner.state.github_text.as_ref().unwrap().resolved);
18971        // Resumed again: the saved decision is applied, nothing is re-asked.
18972        let again = runner
18973            .guarded_pr_message(&winner, None, true)
18974            .await
18975            .unwrap()
18976            .unwrap();
18977        assert_eq!(again.title, message.title);
18978        assert_eq!(
18979            store
18980                .list()
18981                .into_iter()
18982                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18983                .count(),
18984            2
18985        );
18986    }
18987
18988    #[test]
18989    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18990        // What magi itself writes stays English under any configured language,
18991        // so a future localisation of these headings fails here. (The agents'
18992        // own text is also checked by the posting gate.)
18993        let mut state = state_with_summary(
18994            "add retries",
18995            "TITLE: fix(web): batch reads\n- reads run.json once",
18996        );
18997        state.config.graph.language = "ja".to_owned();
18998        let m = pr_message(&state, 'A');
18999        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
19000        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
19001
19002        // The task is the operator's own text: it goes in untouched, and the
19003        // fallback title (no summary) may be in its language too.
19004        let task = "今回やってほしいこと: results projector を直す";
19005        let mut state = state_with_summary(task, "- no title line");
19006        state.config.graph.language = "ja".to_owned();
19007        let m = pr_message(&state, 'A');
19008        assert_eq!(
19009            m.title,
19010            format!("chore: land candidate A of run {}", state.id)
19011        );
19012        assert!(
19013            m.body.contains(&format!(
19014                "<summary>Original task</summary>\n\n{task}\n\n</details>"
19015            )),
19016            "{}",
19017            m.body
19018        );
19019    }
19020
19021    #[test]
19022    fn pr_message_scrubs_home_paths_and_addresses() {
19023        let state = state_with_summary(
19024            "fix it in /Users/someone/src/x",
19025            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
19026        );
19027        let m = pr_message(&state, 'A');
19028        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
19029            assert!(!m.body.contains(leak), "{}", m.body);
19030        }
19031        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
19032    }
19033
19034    #[test]
19035    fn pr_message_survives_a_task_that_closes_details() {
19036        let state = state_with_summary("a </details> b", "TITLE: fix: x");
19037        let m = pr_message(&state, 'A');
19038        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
19039    }
19040
19041    #[test]
19042    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
19043        let cmd = manual_merge_command(
19044            MergeStyle::Squash,
19045            Path::new("/repo"),
19046            "b",
19047            "fix: \"quoted\" $(x) `y`\n\nbody",
19048        );
19049        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
19050    }
19051
19052    #[test]
19053    fn manual_merge_command_matches_the_configured_style() {
19054        let repo = Path::new("/repo");
19055        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
19056
19057        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
19058        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
19059
19060        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
19061        assert_eq!(
19062            squash,
19063            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
19064             \"Merge magi run 0832 (candidate A)\""
19065        );
19066
19067        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
19068        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
19069    }
19070
19071    #[test]
19072    fn a_nudge_gets_a_quarter_of_the_budget() {
19073        // The judge and implement budgets magi ships with.
19074        assert_eq!(retry_budget(secs(1200), true), secs(300));
19075        assert_eq!(retry_budget(secs(3600), true), secs(900));
19076    }
19077
19078    #[test]
19079    fn a_resent_prompt_keeps_the_whole_budget() {
19080        // The seat kept no context, so the retry is the original job again and
19081        // shortening it would only guarantee a second failure.
19082        assert_eq!(retry_budget(secs(1200), false), secs(1200));
19083        assert_eq!(retry_budget(secs(60), false), secs(60));
19084    }
19085
19086    #[test]
19087    fn the_floor_never_exceeds_the_original_budget() {
19088        // A short configured timeout must not be *raised* by the floor: the
19089        // operator asked for a bound, and a retry may not outlast the attempt
19090        // it is retrying.
19091        assert_eq!(retry_budget(secs(60), true), secs(60));
19092        assert_eq!(retry_budget(secs(480), true), secs(120));
19093        assert_eq!(retry_budget(secs(0), true), secs(0));
19094    }
19095
19096    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
19097        agent::CommandEvidence {
19098            id: "item1".to_owned(),
19099            description: "cargo test".to_owned(),
19100            exit_code,
19101            result_summary: String::new(),
19102            source: "codex".to_owned(),
19103        }
19104    }
19105
19106    #[test]
19107    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
19108        // No evidence is not the same fact as unconfirmed evidence: a
19109        // backend with no adapter, or a reply that ran no commands at all,
19110        // must not be misread as carrying a dangling job.
19111        assert!(!has_unconfirmed_command(&[]));
19112    }
19113
19114    #[test]
19115    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
19116        // Deliberately not a check on the exit code's *value*: a fixer
19117        // legitimately runs something that fails mid-iteration before it
19118        // succeeds, and that must never by itself reopen a valid report.
19119        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
19120        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
19121        assert!(!has_unconfirmed_command(&[
19122            evidence(Some(0)),
19123            evidence(Some(101))
19124        ]));
19125    }
19126
19127    #[test]
19128    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
19129        assert!(has_unconfirmed_command(&[
19130            evidence(Some(0)),
19131            evidence(None)
19132        ]));
19133    }
19134
19135    #[test]
19136    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
19137        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19138        assert_eq!(
19139            verified_noop_claim(true, &[], text).as_deref(),
19140            Some("already fixed by b32cfc4, on main.")
19141        );
19142    }
19143
19144    #[test]
19145    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
19146        // A timeout or a bad exit code reads as the ordinary loss it is,
19147        // whatever the reply's own prose claims.
19148        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19149        assert!(verified_noop_claim(false, &[], text).is_none());
19150    }
19151
19152    #[test]
19153    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
19154        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19155        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
19156        // A confirmed command alongside the marker is fine.
19157        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
19158    }
19159
19160    #[test]
19161    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
19162        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
19163    }
19164
19165    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
19166    /// pair, in order, labelled A, B, C, ...
19167    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
19168        runner.state.candidates = shape
19169            .iter()
19170            .enumerate()
19171            .map(|(i, &(empty, verified))| Candidate {
19172                index: i,
19173                label: (b'A' + i as u8) as char,
19174                agent: "sonnet".to_owned(),
19175                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
19176                worktree: PathBuf::from(format!("/wt/{i}")),
19177                summary: String::new(),
19178                stat: String::new(),
19179                files: 0,
19180                commits: 0,
19181                empty,
19182                failed: None,
19183                verified_noop: verified.map(str::to_owned),
19184                duration_ms: 0,
19185                folded: false,
19186            })
19187            .collect();
19188    }
19189
19190    #[test]
19191    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
19192        ask_test_home();
19193        let mut runner = runner_at(RunStatus::Implementing);
19194        set_candidates(
19195            &mut runner,
19196            &[
19197                (true, Some("already on main at b32cfc4")),
19198                (true, Some("same fix, see the existing test")),
19199            ],
19200        );
19201
19202        runner
19203            .after_implement()
19204            .expect("a verified no-op is not an error");
19205
19206        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
19207    }
19208
19209    #[test]
19210    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
19211        ask_test_home();
19212        let mut runner = runner_at(RunStatus::Implementing);
19213        // Candidate A declares a verified no-op; candidate B simply wrote
19214        // nothing and said nothing about why. One candidate's claim is not
19215        // the whole run's agreement.
19216        set_candidates(
19217            &mut runner,
19218            &[(true, Some("already on main at b32cfc4")), (true, None)],
19219        );
19220
19221        let err = runner
19222            .after_implement()
19223            .expect_err("an unverified empty candidate must still fail the run");
19224
19225        assert!(
19226            err.to_string().contains("no candidate produced a change"),
19227            "{err}"
19228        );
19229        assert_eq!(runner.state.status, RunStatus::Failed);
19230    }
19231
19232    #[test]
19233    fn after_implement_still_fails_an_ordinary_all_empty_run() {
19234        ask_test_home();
19235        let mut runner = runner_at(RunStatus::Implementing);
19236        set_candidates(&mut runner, &[(true, None), (true, None)]);
19237
19238        let err = runner
19239            .after_implement()
19240            .expect_err("no candidate declared anything; this is an ordinary failure");
19241
19242        assert!(
19243            err.to_string().contains("no candidate produced a change"),
19244            "{err}"
19245        );
19246        assert_eq!(runner.state.status, RunStatus::Failed);
19247    }
19248}
19249
19250/// A graft rollback whose restore could not be confirmed (see
19251/// [`Runner::graft`]).
19252#[derive(Debug)]
19253struct GraftRestoreFailed(String);
19254
19255impl std::fmt::Display for GraftRestoreFailed {
19256    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
19257        write!(
19258            f,
19259            "could not restore the winner to its pre-graft commit: {}",
19260            self.0
19261        )
19262    }
19263}
19264
19265impl std::error::Error for GraftRestoreFailed {}
19266
19267/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
19268/// not the agent: they never draw on an agent's own timeout, and running out of
19269/// them never reads as an agent failure.
19270#[cfg(test)]
19271mod resource_wait_tests {
19272    use super::*;
19273    use std::collections::BTreeMap;
19274
19275    fn sh(id: &str, script: &str) -> AgentSpec {
19276        AgentSpec {
19277            id: id.to_owned(),
19278            kind: crate::config::AgentKind::Command,
19279            model: None,
19280            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
19281            extra_args: Vec::new(),
19282            env: BTreeMap::new(),
19283            prompt_delivery: None,
19284            max_turn_secs: None,
19285        }
19286    }
19287
19288    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
19289        SeatJob {
19290            seat: SeatState::new("impl-A", &spec.id, 1),
19291            spec,
19292            cwd: dir.to_path_buf(),
19293            prompt: "do it".to_owned(),
19294            timeout,
19295            allow_write: true,
19296            sessions: false,
19297            artifacts: dir.join("art"),
19298            stem: "impl-A".to_owned(),
19299            handover: None,
19300        }
19301    }
19302
19303    fn state(resource_wait_secs: u64) -> RunState {
19304        let mut config = Config::default();
19305        config.graph.timeout_resource_wait = resource_wait_secs;
19306        RunState::new(
19307            PathBuf::from("/repo"),
19308            "main".to_owned(),
19309            "abc1234".to_owned(),
19310            "task".to_owned(),
19311            config,
19312        )
19313    }
19314
19315    fn hold(cache: &Path) -> crate::cache::Guard {
19316        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
19317        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
19318            crate::cache::AcquireOutcome::Acquired(g) => g,
19319            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
19320        }
19321    }
19322
19323    async fn run(
19324        jobs: Vec<SeatJob>,
19325        sem: Arc<Semaphore>,
19326        cache: Option<&Path>,
19327        st: &mut RunState,
19328    ) -> Vec<(usize, SeatState, AgentOutcome)> {
19329        let prompts = Prompts::default();
19330        let ctx = WaveCtx {
19331            run: "r",
19332            node: "implement",
19333            prompts: &prompts,
19334            cache,
19335            round: None,
19336            carry_seats: false,
19337            pauses: None,
19338        };
19339        wave(jobs, sem, &ctx, st, 0).await
19340    }
19341
19342    #[tokio::test]
19343    async fn verification_cache_wait_preserves_the_command_budget() {
19344        crate::run::pin_test_home();
19345        let tmp = tempfile::tempdir().unwrap();
19346        let cache = tmp.path().join("target");
19347        let guard = hold(&cache);
19348        let release = tokio::spawn(async move {
19349            tokio::time::sleep(Duration::from_millis(100)).await;
19350            guard.release();
19351        });
19352        let mut st = state(10);
19353        let budget = Duration::from_millis(50);
19354        let wait = repository_wait(&st, None);
19355        let (outcomes, _) = with_cache_lease(
19356            &mut st,
19357            Some(&cache),
19358            "gate",
19359            "gate",
19360            tmp.path(),
19361            "abc1234",
19362            budget,
19363            "test gate",
19364            &wait,
19365            |_, available| async move {
19366                assert_eq!(
19367                    available, budget,
19368                    "contention must not shorten verification"
19369                );
19370                (Vec::new(), false, Vec::new())
19371            },
19372        )
19373        .await
19374        .unwrap();
19375        release.await.unwrap();
19376        assert!(outcomes.is_empty(), "{outcomes:?}");
19377        assert!(
19378            st.events
19379                .iter()
19380                .any(|event| event.message.contains("waiting for"))
19381        );
19382        assert!(st.resource_wait.is_none());
19383        // The verification body no longer owns the lease after returning.
19384        hold(&cache).release();
19385    }
19386
19387    #[tokio::test]
19388    async fn verification_cache_wait_expires_without_starting_commands() {
19389        crate::run::pin_test_home();
19390        let tmp = tempfile::tempdir().unwrap();
19391        let cache = tmp.path().join("target");
19392        let _held = hold(&cache);
19393        let mut st = state(0);
19394        let wait = repository_wait(&st, None);
19395        let (outcomes, _) = with_cache_lease(
19396            &mut st,
19397            Some(&cache),
19398            "gate",
19399            "gate",
19400            tmp.path(),
19401            "abc1234",
19402            Duration::from_secs(60),
19403            "test gate",
19404            &wait,
19405            |_, _| async { panic!("commands must not start without the cache lease") },
19406        )
19407        .await
19408        .unwrap();
19409        assert_eq!(outcomes.len(), 1);
19410        assert!(outcomes[0].resource_blocked);
19411        assert!(st.resource_wait.is_none());
19412    }
19413
19414    #[tokio::test]
19415    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
19416        crate::run::pin_test_home();
19417        let tmp = tempfile::tempdir().unwrap();
19418        let cache = tmp.path().join("target-a");
19419        let guard = hold(&cache);
19420        // Held for longer than the agent's whole 1s budget.
19421        tokio::spawn(async move {
19422            tokio::time::sleep(Duration::from_millis(1500)).await;
19423            guard.release();
19424        });
19425        let mut st = state(60);
19426        let results = run(
19427            vec![job(
19428                sh("alpha", "echo ok"),
19429                tmp.path(),
19430                Duration::from_secs(1),
19431            )],
19432            Arc::new(Semaphore::new(1)),
19433            Some(&cache),
19434            &mut st,
19435        )
19436        .await;
19437        assert!(
19438            matches!(results[0].2, AgentOutcome::Ok(_)),
19439            "the wait must not be charged to the agent"
19440        );
19441        assert!(st.resource_block.is_none());
19442        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19443        assert!(st.resource_wait.is_none() && st.active.is_empty());
19444    }
19445
19446    #[tokio::test]
19447    #[cfg(unix)]
19448    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19449        use std::os::fd::AsRawFd as _;
19450        crate::run::pin_test_home();
19451        let tmp = tempfile::tempdir().unwrap();
19452        crate::git::git(tmp.path(), &["init"]).await.unwrap();
19453        let common = tmp.path().join(".git");
19454        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19455        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19456        assert_eq!(
19457            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19458            0
19459        );
19460        let mut st = state(1);
19461        st.config.coordination = crate::config::test_coordination();
19462        let mut reader = job(
19463            sh("reader", "echo must-not-run"),
19464            tmp.path(),
19465            Duration::from_secs(30),
19466        );
19467        reader.allow_write = false;
19468        let results = run(
19469            vec![reader.clone()],
19470            Arc::new(Semaphore::new(1)),
19471            None,
19472            &mut st,
19473        )
19474        .await;
19475        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19476        assert_eq!(results[0].1.turns, 0);
19477        assert!(FailClass::of(&results[0].2).is_none());
19478        assert_eq!(
19479            st.resource_block.as_ref().unwrap().kind,
19480            "repository coordination"
19481        );
19482        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19483        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19484        drop(external);
19485        let editor = crate::repo_lock::SeatLease::for_checkout(
19486            tmp.path(),
19487            true,
19488            crate::config::test_coordination().as_ref(),
19489        )
19490        .await
19491        .unwrap();
19492        let pause = Pause::new();
19493        let interrupt = Pause::new();
19494        let prompts = Prompts::default();
19495        let ctx = WaveCtx {
19496            run: "r",
19497            node: "review",
19498            prompts: &prompts,
19499            cache: None,
19500            round: None,
19501            carry_seats: false,
19502            pauses: Some((&pause, &interrupt)),
19503        };
19504        let (results, ()) = tokio::join!(
19505            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19506            async {
19507                tokio::time::sleep(Duration::from_millis(50)).await;
19508                pause.park();
19509            }
19510        );
19511        assert!(
19512            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19513        );
19514        assert_eq!(results[0].1.turns, 0);
19515        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19516        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19517        drop(editor);
19518    }
19519
19520    #[tokio::test]
19521    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19522        crate::run::pin_test_home();
19523        let tmp = tempfile::tempdir().unwrap();
19524        let cache = tmp.path().join("target-b");
19525        let _held = hold(&cache);
19526        let mut st = state(1);
19527        let results = run(
19528            vec![job(
19529                sh("alpha", "echo ok"),
19530                tmp.path(),
19531                Duration::from_secs(1),
19532            )],
19533            Arc::new(Semaphore::new(1)),
19534            Some(&cache),
19535            &mut st,
19536        )
19537        .await;
19538        let (_, seat, out) = &results[0];
19539        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19540        assert!(wave_blocked(&results));
19541        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19542        assert_eq!(seat.turns, 0, "nothing was asked");
19543        assert!(
19544            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19545            "no agent was spawned"
19546        );
19547        assert!(FailClass::of(out).is_none(), "not a failure class");
19548        let block = st.resource_block.as_ref().expect("recorded for the resume");
19549        assert_eq!(block.kind, "shared build cache");
19550        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19551        assert!(st.active.is_empty());
19552    }
19553
19554    #[tokio::test]
19555    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19556        crate::run::pin_test_home();
19557        let tmp = tempfile::tempdir().unwrap();
19558        let mut st = state(1);
19559        let results = run(
19560            vec![job(
19561                sh("alpha", "echo ok"),
19562                tmp.path(),
19563                Duration::from_secs(1),
19564            )],
19565            Arc::new(Semaphore::new(0)),
19566            None,
19567            &mut st,
19568        )
19569        .await;
19570        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19571        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19572        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19573    }
19574
19575    #[tokio::test]
19576    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19577        crate::run::pin_test_home();
19578        let tmp = tempfile::tempdir().unwrap();
19579        let mut st = state(60);
19580        let results = run(
19581            vec![job(
19582                sh("alpha", "sleep 30"),
19583                tmp.path(),
19584                Duration::from_secs(1),
19585            )],
19586            Arc::new(Semaphore::new(1)),
19587            None,
19588            &mut st,
19589        )
19590        .await;
19591        let out = &results[0].2;
19592        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19593        let class = FailClass::of(out).expect("a failure");
19594        assert_eq!(class, FailClass::Timeout);
19595        assert!(should_hand_over(None, &class), "existing handover policy");
19596        assert!(st.resource_block.is_none());
19597    }
19598}