Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim shared by targeted repairs and repository reassignment.
534/// Held before asynchronous fix preflight and released on every return or panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549pub(crate) struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Capture task identity and repository assignment before the first
773        // save, so publication scope survives crashes and later task moves.
774        state.fetched_base = Some(fetched_base);
775        state.set_origin(origin)?;
776        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777        state.status = RunStatus::Prep;
778        state.event("start", format!("run {} created", state.id));
779        startup.finish(&mut state, Ok(()))?;
780        Ok(Self {
781            state,
782            roles,
783            sem: Arc::new(Semaphore::new(max_parallel)),
784            pause: Pause::new(),
785            interrupt: Pause::new(),
786        })
787    }
788
789    /// Open a review-only run against work that already exists on `branch`.
790    ///
791    /// The expensive half of the graph is the implement wave — measured at
792    /// 111 and 134 internal tool-loop turns on this repository, against a
793    /// handful for a judge or a reviewer. The cheap half is worth running on
794    /// hand-written work too, and there was no way to reach it.
795    ///
796    /// No new state and no schema change are needed: a run with **one** viable
797    /// candidate and a tally already decided degrades `execute` to exactly
798    /// review → gate → merge, because `judge` skips a single-candidate field,
799    /// `deliberate` has fewer than two first choices to reconcile, `vote`
800    /// returns early, `tally` is already present and `fold_losers` has no
801    /// losers. Resuming such a run therefore does the right thing as well.
802    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803        Self::review_taking_over(repo, branch, config, None, origin).await
804    }
805
806    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
807    /// at the same task still has `branch` checked out, its worktree is
808    /// released first if that is safe (see [`crate::handover`]), and the
809    /// review refuses with the reason if it is not. `None` is a hand-run
810    /// review: it has no earlier attempts, so only a worktree of a dead run
811    /// magi recorded itself can be released.
812    pub async fn review_taking_over(
813        repo: &Path,
814        branch: &str,
815        config: Config,
816        takeover: Option<crate::handover::Takeover>,
817        origin: Origin,
818    ) -> Result<Self> {
819        let repo = git::toplevel(repo).await?;
820        let missing = agent::missing_programs(&config.agents);
821        if !missing.is_empty() {
822            bail!(
823                "these agent programs are not on PATH: {}. Fix the roster in \
824                 magi.toml or install them.",
825                missing.join(", ")
826            );
827        }
828        let base_branch =
829            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830                .await?;
831        if base_branch == branch {
832            bail!("`{branch}` is the base branch; there is nothing to review against");
833        }
834        let fetched_base = resolve_base(
835            &repo,
836            &base_branch,
837            &config.merge.remote,
838            config.coordination.as_ref(),
839        )
840        .await?;
841        let base_commit = fetched_base.commit.clone();
842
843        let roles = config.resolve_roles()?;
844        let max_parallel = config.graph.max_parallel.max(1);
845        let mut state = RunState::new(
846            repo.clone(),
847            base_branch,
848            base_commit.clone(),
849            format!("Review the work already on branch `{branch}`."),
850            config,
851        );
852        state.fetched_base = Some(fetched_base);
853        state.set_origin(origin)?;
854        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856        let opened = async {
857            startup.stage(&mut state, "handover")?;
858
859            // Released before anything else touches the branch: a stale local
860            // branch is moved with `git branch -f`, which git refuses while an
861            // earlier attempt's worktree still has it checked out. Everything
862            // after this point that can fail puts the old run back.
863            // A hand-run review has no task, hence no earlier attempts, but a
864            // worktree of a dead run magi made may still be released.
865            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866                earlier: Vec::new(),
867                home: crate::run::home(),
868                choice: None,
869                task_instruction: None,
870            });
871            let wait = repository_wait(&state, None);
872            let coordination_config = state.config.coordination.clone();
873            let coordination =
874                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875                    .await?;
876            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877            if let Some(released) = &released {
878                state.event(
879                    "release",
880                    format!(
881                        "took `{branch}` over from run {}: its worktree was released: {}",
882                        crate::run::short_of(&released.old_id),
883                        released.audit
884                    ),
885                );
886            }
887            let opened = async {
888                startup.stage(&mut state, "review-preflight")?;
889                // The owner's answer to an earlier divergence question is applied
890                // here: after the release (git will not move a checked-out branch)
891                // and before the sync that would otherwise ask again.
892                state.foreign_checkout = crate::handover::preflight_review(
893                    &repo,
894                    branch,
895                    &takeover.home,
896                    &state.config.merge.remote,
897                )
898                .await?;
899                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901                }
902                if let Some(choice) = takeover.choice.as_ref()
903                    && let Err(e) =
904                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905                            .await
906                {
907                    return Err(e.context("applying the owner's answer about the diverged branch"));
908                }
909                // An earlier attempt of this task that published the task head, and
910                // whose work is the branch under review, hands that head over: the
911                // review run updates the same pull request. Without one (a legacy
912                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
913                // own head and its own open pull request is adopted as before.
914                for id in takeover.earlier.iter().rev() {
915                    if let Ok(old) = RunState::load_under(id, &takeover.home)
916                        && let Some(head) = old.pr_head.clone()
917                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918                    {
919                        state.pr_head = Some(head);
920                        break;
921                    }
922                }
923                drop(coordination);
924                startup.stage(&mut state, "open-review")?;
925                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926            }
927            .await;
928            if opened.is_err()
929                && let Some(released) = &released
930            {
931                // Restoration is ownership checked and best effort. Keep the original
932                // startup cause even if reacquiring coordination is no longer possible.
933                match crate::repo_lock::SeatLease::operation_with(
934                    &repo,
935                    coordination_config.as_ref(),
936                    &wait,
937                )
938                .await
939                {
940                    Ok(_coordination) => {
941                        released.restore(&repo, branch).await;
942                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
943                            .map(|old| {
944                                format!(
945                                    "predecessor released_to={:?}; candidate path exists={}",
946                                    old.released_to,
947                                    old.candidates.iter().any(|c| {
948                                        c.branch == branch && c.worktree.exists()
949                                    }),
950                                )
951                            })
952                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954                    }
955                    Err(error) => {
956                        tracing::warn!(
957                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958                            state.id
959                        );
960                        state.event(
961                            "startup-restore",
962                            format!("restoration not attempted: {error:#}"),
963                        );
964                    }
965                }
966            }
967            opened
968        }
969        .await;
970        startup.finish(&mut state, opened)?;
971        Ok(Self {
972            state,
973            roles,
974            sem: Arc::new(Semaphore::new(max_parallel)),
975            pause: Pause::new(),
976            interrupt: Pause::new(),
977        })
978    }
979
980    /// Recover exactly the operator-selected public implementation, without touching old sources.
981    pub async fn recover_pr(
982        repo: &Path,
983        task: &str,
984        selection: crate::pr_recovery::Selection,
985        mut config: Config,
986    ) -> Result<Self> {
987        selection.configure(&mut config)?;
988        let assignment = crate::run::task_assignment_for(task)?;
989        selection.check_task_assignment(task, &selection.instruction, assignment)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.set_origin(crate::run::Origin::queue(task))?;
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256            Err(e) => Err(e),
1257        };
1258        if let Err(e) = setup {
1259            git::worktree_remove_clean(repo, &worktree).await.ok();
1260            return Err(e);
1261        }
1262        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263            .await
1264            .unwrap_or(0);
1265        if commits == 0 {
1266            git::worktree_remove_clean(repo, &worktree).await.ok();
1267            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268        }
1269        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270            .await
1271            .map(|f| f.len())
1272            .unwrap_or(0);
1273        if files == 0
1274            && let (Ok(head_tree), Ok(base_tree)) = (
1275                git::tree_of(&worktree, "HEAD").await,
1276                git::tree_of(&worktree, &base_commit).await,
1277            )
1278            && head_tree == base_tree
1279        {
1280            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281            git::worktree_remove_clean(repo, &worktree).await.ok();
1282            bail!(
1283                "`{branch}` at {} has a tree identical to base {}; this usually means \
1284                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285                 against `{}/{branch}`) rather than an empty change",
1286                short(&head),
1287                short(&base_commit),
1288                state.config.merge.remote
1289            );
1290        }
1291        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292            .await
1293            .unwrap_or_default();
1294
1295        state.candidates.push(Candidate {
1296            index: 0,
1297            label: 'A',
1298            // Not an agent id on purpose: nothing in the roster wrote this, and
1299            // the stats tables must not credit anyone with a win for it.
1300            agent: EXISTING_BRANCH.to_owned(),
1301            branch: review_branch,
1302            worktree,
1303            summary: String::new(),
1304            stat,
1305            files,
1306            commits,
1307            empty: false,
1308            failed: None,
1309            verified_noop: None,
1310            duration_ms: 0,
1311            folded: false,
1312        });
1313        state.tally = Some(Tally {
1314            first_choice: BTreeMap::from([('A', 0)]),
1315            borda: BTreeMap::new(),
1316            winner: 'A',
1317            rankings: 0,
1318            unanimous_initial: false,
1319            deliberated: false,
1320            changed_votes: 0,
1321            unanimous_final: false,
1322            tie_break: None,
1323            // No panel sat, so no quorum applies. Zero judges is the correct
1324            // number for work that never competed, and must not be reported as
1325            // a collapsed panel.
1326            judges: 0,
1327            present: 0,
1328            quorum: 0,
1329            met_quorum: true,
1330            uncontested: Some("review-only run: nothing competed".to_owned()),
1331        });
1332        state.status = RunStatus::Reviewing;
1333        state.prep_complete = Some(true);
1334        state.event(
1335            "start",
1336            format!(
1337                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338                state.id
1339            ),
1340        );
1341        Ok(())
1342    }
1343
1344    /// Authorize one bounded synchronization interval, under the caller's queue task
1345    /// claim and the same owner/execution locks used by run maintenance.
1346    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347        Self::authorize_base_recovery_request(home, id, task, false, None).await
1348    }
1349
1350    /// Queue release uses its durable approval generation exactly once.
1351    pub(crate) async fn authorize_base_recovery_from_resumption(
1352        home: &Path,
1353        id: &str,
1354        task: &str,
1355    ) -> Result<()> {
1356        Self::authorize_base_recovery_request(home, id, task, true, None).await
1357    }
1358
1359    pub(crate) async fn authorize_base_recovery_control(
1360        home: &Path,
1361        task: &str,
1362        control: &mut crate::base_recovery_control::Transaction,
1363    ) -> Result<()> {
1364        let id = control.request.run.clone();
1365        Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1366    }
1367
1368    async fn authorize_base_recovery_request(
1369        home: &Path,
1370        id: &str,
1371        task: &str,
1372        resumption: bool,
1373        mut control: Option<&mut crate::base_recovery_control::Transaction>,
1374    ) -> Result<()> {
1375        let _owner = crate::pr_head::owner_lock(home, id)?;
1376        let mut state = RunState::load_under(id, home)?;
1377        crate::pr_head::ensure_owner(&state)?;
1378        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1379        anyhow::ensure!(
1380            queued.execution_runs().last().is_some_and(|run| run == id)
1381                && queued.status != crate::queue::TaskStatus::Done
1382                && !queued.fresh_start
1383                && queued.review_branch.is_none(),
1384            "run task ownership changed"
1385        );
1386        anyhow::ensure!(
1387            state
1388                .origin
1389                .as_ref()
1390                .and_then(|o| o.task.as_deref())
1391                .is_none_or(|owner| owner == task),
1392            "run belongs to another task"
1393        );
1394        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1395        anyhow::ensure!(
1396            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1397            "run is live or its ownership is unknown"
1398        );
1399        // Judge the previous driver's lease before acquiring our maintenance
1400        // lease. The owner lock excludes new driver claims across both steps.
1401        let _execution = crate::driver::maintenance(home, id)?;
1402        anyhow::ensure!(
1403            state.status.resumable()
1404                && state.released_to.is_none()
1405                && state.retired_to.is_none()
1406                && state.pr_taken_over_by.is_none(),
1407            "run ownership is final"
1408        );
1409        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1410        anyhow::ensure!(
1411            !state.pr_publication_pending
1412                && !crate::handover::landing_checkpoint(&state)
1413                && state.merge_decision.is_none(),
1414            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1415        );
1416        let winner = state.winner().context("run has no retained winner")?;
1417        anyhow::ensure!(
1418            git::is_clean(&winner.worktree).await?,
1419            "retained worktree has uncommitted changes"
1420        );
1421        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1422        anyhow::ensure!(
1423            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1424            "retained worktree does not match the winner branch"
1425        );
1426        if let Some(control) = control.as_ref() {
1427            anyhow::ensure!(
1428                head.starts_with(&control.request.head),
1429                "retained HEAD changed since the owner request"
1430            );
1431            anyhow::ensure!(
1432                state
1433                    .base_sync_recovery
1434                    .stop
1435                    .as_ref()
1436                    .is_none_or(|stop| stop.head == head),
1437                "retained HEAD differs from the recorded stop head; inspect it first"
1438            );
1439        }
1440        state.ensure_base_episode();
1441        state.record_base_sync_stop(&head);
1442        state.save_under(home)?;
1443        let fetched =
1444            git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1445        anyhow::ensure!(
1446            fetched.ok(),
1447            "cannot authorize base recovery without a fetched base"
1448        );
1449        let approved_tip = git::rev_parse(
1450            &state.repo,
1451            &format!("{}/{}", state.config.merge.remote, state.base_branch),
1452        )
1453        .await?;
1454        if let Some(control) = control.as_deref_mut() {
1455            let queue = crate::queue::Queue::at(home.join("queue"));
1456            control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1457            if let Some(prior) = state
1458                .base_sync_recovery
1459                .authorizations
1460                .iter()
1461                .find(|a| a.request.as_deref() == Some(&control.key()))
1462            {
1463                anyhow::ensure!(
1464                    !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1465                    "recovery request was consumed or invalidated; it cannot queue again"
1466                );
1467                return control.enqueue(&queue);
1468            }
1469        }
1470        if resumption {
1471            anyhow::ensure!(
1472                crate::daemon::compatible_run_instruction(&state, &queued),
1473                "resumption instruction does not match the retained run"
1474            );
1475            let approval = queued
1476                .resumption
1477                .as_ref()
1478                .context("no operator resumption")?;
1479            anyhow::ensure!(
1480                approval.instruction == queued.instruction,
1481                "resumption approval does not match the current task instruction"
1482            );
1483            state.authorize_base_sync_resumption(task, approval)?;
1484        } else {
1485            state.authorize_base_sync(task)?;
1486        }
1487        let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1488        state
1489            .base_sync_recovery
1490            .authorizations
1491            .last_mut()
1492            .unwrap()
1493            .scope = Some(crate::base_sync_recovery::Scope {
1494            tip: approved_tip,
1495            instruction: queued.instruction.clone(),
1496            episode,
1497        });
1498        if let Some(control) = control.as_deref_mut() {
1499            let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1500            authorization.request = Some(control.key());
1501            authorization.generation = control.generation();
1502            authorization.decision = control.decision();
1503            authorization.question = control
1504                .request
1505                .decision
1506                .as_ref()
1507                .map(|_| control.request.request.clone());
1508        }
1509        state.event(
1510            "base-recovery",
1511            "owner authorized a bounded base synchronization interval; fresh verification required",
1512        );
1513        state.save_under(home)?;
1514        if let Some(control) = control {
1515            control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1516        }
1517        Ok(())
1518    }
1519
1520    /// Reopen an existing run.
1521    pub fn resume(id: &str) -> Result<Self> {
1522        let state = RunState::load(id)?;
1523        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1524        if state.startup.as_ref().is_some_and(|s| !s.complete)
1525            || (state.status == RunStatus::Prep
1526                && state.instruction.trim().is_empty()
1527                && state.candidates.is_empty()
1528                && state.prep_complete == Some(false))
1529        {
1530            bail!(
1531                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1532            );
1533        }
1534        crate::pr_head::ensure_owner(&state)?;
1535        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1536            bail!(
1537                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1538            );
1539        }
1540        if let Some(to) = &state.released_to {
1541            bail!(
1542                "run {} cannot be resumed: its worktree was released to run {}",
1543                state.short(),
1544                crate::run::short_of(to)
1545            );
1546        }
1547        let roles = state.config.resolve_roles()?;
1548        if let Some(id) = &state.config.graph.selected_implementer
1549            && state.candidates.iter().any(|c| c.agent != *id)
1550        {
1551            bail!(
1552                "replacement implementer pin does not match the saved candidates; task must stay held"
1553            );
1554        }
1555        let max_parallel = state.config.graph.max_parallel.max(1);
1556        Ok(Self {
1557            state,
1558            roles,
1559            sem: Arc::new(Semaphore::new(max_parallel)),
1560            pause: Pause::new(),
1561            interrupt: Pause::new(),
1562        })
1563    }
1564
1565    /// Walk the graph to a terminal state, skipping nodes already recorded.
1566    ///
1567    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1568    /// the phone - ends here, so this is the one place a run that ended
1569    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1570    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1571    pub fn execute(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
1572        // Allocate before polling: boxing inside an async wrapper would still
1573        // retain its construction frame while the nested graph is polled.
1574        Box::pin(self.execute_inner())
1575    }
1576
1577    async fn execute_inner(&mut self) -> Result<()> {
1578        anyhow::ensure!(
1579            self.state.startup.as_ref().is_none_or(|s| s.complete),
1580            "startup incomplete; inspect retained evidence before explicit recovery"
1581        );
1582        if let Some(selection) = &self.state.pr_recovery
1583            && (selection.source_proof.is_some() || selection.source_run.is_some())
1584        {
1585            Box::pin(selection.verify_source(&self.state.repo)).await?;
1586        }
1587        // Queue claims cover daemon work; this also serializes manual drivers
1588        // consuming a repair answer, including the new review and decision.
1589        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1590            || self.state.land_approval.is_some()
1591            || self.state.merge_decision.is_some()
1592            || !self.state.merge_repairs.is_empty()
1593            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1594        {
1595            let claim = FixClaim::acquire(&self.state.dir())?;
1596            crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1597            let disk = RunState::load(&self.state.id)?;
1598            let checkpoint = |s: &RunState| {
1599                (
1600                    s.merge_repairs.len(),
1601                    s.merge_repairs
1602                        .last()
1603                        .map(|r| (r.question.clone(), r.phase)),
1604                    s.land_approval
1605                        .as_ref()
1606                        .map(|a| (a.question.clone(), a.head.clone())),
1607                )
1608            };
1609            if checkpoint(&disk) != checkpoint(&self.state) {
1610                bail!(
1611                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1612                );
1613            }
1614            Some(claim)
1615        } else {
1616            None
1617        };
1618        let mut execution = self.claim_driver().await?;
1619        let runner = &mut *execution.runner;
1620        let wait = runner.repository_wait();
1621        // Keep the graph's large future off the callers' stack, including
1622        // standalone task attempts running on a default-sized test thread.
1623        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1624        let result = match result {
1625            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1626                record_repository_block(&mut runner.state, &e);
1627                runner.park_here()?;
1628                Ok(())
1629            }
1630            other => other,
1631        };
1632        execution.driver.finish(&mut runner.state).await?;
1633        let ended = if result.is_err() {
1634            Some({
1635                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1636                if let Err(error) = &result {
1637                    notice.message = format!(
1638                        "Run {} stopped with an error: {}",
1639                        runner.state.id,
1640                        error.to_string().lines().next().unwrap_or("unknown error")
1641                    );
1642                }
1643                notice
1644            })
1645        } else {
1646            crate::notices::run_ended(&runner.state)
1647        };
1648        if let Some(notice) = ended {
1649            crate::notices::raise(notice);
1650        }
1651        result
1652    }
1653
1654    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1655        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1656    }
1657
1658    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1659        let wait = self.repository_wait();
1660        coordinate_operation(&mut self.state, cwd, &wait).await
1661    }
1662
1663    /// Validate ownership and persist the driver claim in one critical section.
1664    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1665        let owner = match crate::pr_head::owner_lock_wait(
1666            &crate::run::home(),
1667            &self.state.id,
1668            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1669        )
1670        .await
1671        {
1672            Ok(owner) => owner,
1673            Err(error) => {
1674                let detail = format!("worktree maintenance or ownership update: {error:#}");
1675                // Admission did not own run.json: retain this resource outcome
1676                // in memory for queue settlement without overwriting a concurrent
1677                // retirement or publication transaction.
1678                self.state.resource_block = Some(crate::run::ResourceBlock {
1679                    cause: crate::run::ResourceBlockCause::Other,
1680                    node: "start".into(),
1681                    kind: "worktree maintenance".into(),
1682                    detail: detail.clone(),
1683                    at: Timestamp::now(),
1684                });
1685                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1686            }
1687        };
1688        let disk = RunState::load(&self.state.id)?;
1689        crate::pr_head::ensure_owner(&disk)?;
1690        let claimed =
1691            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1692        // Only proven-live ownership refuses here. Unknown records (legacy runs
1693        // with no claim marker or process identity) fall through to the
1694        // exclusive execution lease taken just below, which is what actually
1695        // keeps a second driver out.
1696        anyhow::ensure!(
1697            disk.liveness(claimed) != crate::run::Liveness::Live,
1698            "run already has a live driver; wait for its execution to end"
1699        );
1700        if let Some(to) = &disk.released_to {
1701            bail!(
1702                "run {} cannot continue: its worktree was released to run {}",
1703                self.state.short(),
1704                crate::run::short_of(to)
1705            );
1706        }
1707        anyhow::ensure!(
1708            serde_json::to_value(&disk.base_sync_recovery)?
1709                == serde_json::to_value(&self.state.base_sync_recovery)?,
1710            "base recovery changed since this run was loaded; reload the saved run"
1711        );
1712        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1713        let mut execution = DriverExecution {
1714            runner: self,
1715            driver,
1716        };
1717        // Declare after the execution guard: on any claim error this short
1718        // lock must drop before that guard reloads and releases ownership.
1719        let _owner = owner;
1720        crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1721            .await?;
1722        let state = &mut execution.runner.state;
1723        // Moving again, so it is no longer parked. Set before the walk rather
1724        // than in `resume`, so every way of re-entering the graph clears it
1725        // and a card cannot claim a run is waiting to be resumed while the
1726        // agents are already working.
1727        state.parked = false;
1728        state.resource_wait = None;
1729        state.resource_block = None;
1730        // Any seat this state still lists as answering belongs to whatever
1731        // process last drove this run — this one included, if it crashed
1732        // mid-wave. Cleared and flushed immediately, before anything else
1733        // runs, so a resume can never show a seat as live when nothing is
1734        // asking it anything yet; the node that actually dispatches the next
1735        // wave repopulates it.
1736        state.clear_active();
1737        // Recorded in the same spot, and flushed together with the clear
1738        // above: this is the pid a reader checks (`RunState::liveness`) when
1739        // no daemon claim exists to answer "is a process still driving this
1740        // run" — a plain `magi run` / `magi review` typed into a terminal
1741        // claims nothing there. Always overwritten, never only-if-absent, so
1742        // a resumed run's stale pid from a previous, possibly-dead process
1743        // can never survive into this one's own report. Unlike
1744        // `clear_active`, this changes on every single `execute()` call, so
1745        // the save below is now unconditional rather than only-if-cleared.
1746        //
1747        // `driver_started_at` is recorded in the same breath, from this same
1748        // pid, so `liveness` can tell a live pid that is genuinely still us
1749        // apart from one the OS has since handed to an unrelated process —
1750        // see that field's own doc for why the pid alone is not enough.
1751        let pid = std::process::id();
1752        state.driver_pid = Some(pid);
1753        state.driver_started_at = crate::proc::process_started_at(pid);
1754        state.driver_exited = false;
1755        state.driver_claimed = Some(true);
1756        state.begin_pass();
1757        execution.driver.bind(state);
1758        state.save()?;
1759        execution.driver.record(state)?;
1760        Ok(execution)
1761    }
1762
1763    /// No conflict checkpoint proves dispatch was never reached. End that
1764    /// consumed generation so an owner can explicitly authorize a fresh one.
1765    async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1766        let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1767        if let Err(error) = &result
1768            && !error.is::<crate::repo_lock::ResourceBlocked>()
1769            && self
1770                .state
1771                .repair_grants
1772                .last()
1773                .is_some_and(|g| g.conflict.is_none())
1774        {
1775            self.state.repair_grants.last_mut().unwrap().phase =
1776                crate::repair_budget::Phase::Complete;
1777            self.state.status = RunStatus::Blocked;
1778            self.state.save()?;
1779            bail!(
1780                "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1781                self.state.passes.last().map_or(0, |p| p.index)
1782            );
1783        }
1784        result
1785    }
1786
1787    /// A grant owns only these repair attempts. Confirmation reviews cannot
1788    /// create another fixer allowance, and every operation is restart-checkpointed.
1789    async fn execute_granted_repair(&mut self) -> Result<()> {
1790        use crate::repair_budget::{Kind, Phase};
1791        let index = self.state.repair_grants.len() - 1;
1792        let winner = self
1793            .state
1794            .winner()
1795            .cloned()
1796            .context("repair winner missing")?;
1797        loop {
1798            let grant = self.state.repair_grants[index].clone();
1799            match grant.phase {
1800                Phase::Pending => {
1801                    if grant.spent >= grant.receipt.request.rounds {
1802                        self.state.repair_grants[index].phase = Phase::Complete;
1803                        self.state.status = RunStatus::Blocked;
1804                        self.state.save()?;
1805                        return Ok(());
1806                    }
1807                    // Reserve before dispatch. A crash can consume an attempt,
1808                    // but can never duplicate it or reset saved fixer history.
1809                    self.state.repair_grants[index].spent += 1;
1810                    self.state.repair_grants[index].phase = Phase::Fixing;
1811                    self.state.save()?;
1812                    match grant.receipt.request.kind {
1813                        Kind::BaseConflict => self.repair_base_conflict(true).await?,
1814                        Kind::Gate => {
1815                            let failed = self.state.gate.clone();
1816                            let _ = self.gate_fix_round(&winner, &failed).await?;
1817                        }
1818                        Kind::Review => {
1819                            let mut record = self
1820                                .state
1821                                .reviews
1822                                .last()
1823                                .cloned()
1824                                .context("review failure missing")?;
1825                            let round = record.round;
1826                            let step = self.run_fix_round(&winner, &mut record).await?;
1827                            self.state.reviews[round - 1] = record;
1828                            if matches!(step, FixStep::Recorded { report_lost: true }) {
1829                                self.state.repair_grants[index].phase = Phase::Complete;
1830                                self.state.status = RunStatus::Blocked;
1831                                self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1832                                self.state.save()?;
1833                                return Ok(());
1834                            }
1835                            if matches!(step, FixStep::Resource) {
1836                                self.stop_for_resource()?;
1837                                return Ok(());
1838                            }
1839                            self.state.save()?;
1840                        }
1841                    }
1842                    if self.resource_halt()? {
1843                        return Ok(());
1844                    }
1845                }
1846                Phase::Fixing => {
1847                    if grant.receipt.request.kind == Kind::BaseConflict {
1848                        self.repair_base_conflict(false).await?;
1849                    }
1850                    // Interrupted fixers may have left source changes or an
1851                    // unpushed commit. Preserve them before fresh assessment.
1852                    let _coordination = self.coordinate(&winner.worktree).await?;
1853                    let rescued = git::rescue_commit(
1854                        &winner.worktree,
1855                        "magi: preserve interrupted granted repair",
1856                    )
1857                    .await?;
1858                    self.state.note_withheld("repair-grant", &rescued.withheld);
1859                    // Fresh checks must supersede even a same-head failure.
1860                    // Archive rather than erase failure evidence or counters.
1861                    self.state
1862                        .verification_history
1863                        .push(crate::run::VerificationEvidence {
1864                            superseded_by: None,
1865                            archived_at: Some(Timestamp::now()),
1866                            fixes_spent: Some(self.state.gate_fixes.len()),
1867                            gate_head: self.state.gate_head.take(),
1868                            gate_ran: std::mem::take(&mut self.state.gate_ran),
1869                            gate: std::mem::take(&mut self.state.gate),
1870                            e2e: self.state.head_e2e.take(),
1871                        });
1872                    if let Some(e2e) = self.state.gate_repair_e2e.take() {
1873                        self.state
1874                            .verification_history
1875                            .push(crate::run::VerificationEvidence {
1876                                superseded_by: None,
1877                                archived_at: Some(Timestamp::now()),
1878                                fixes_spent: Some(self.state.gate_fixes.len()),
1879                                gate_head: None,
1880                                gate_ran: false,
1881                                gate: Vec::new(),
1882                                e2e: Some(e2e),
1883                            });
1884                    }
1885                    self.state.repair_grants[index].review_start = self.state.reviews.len();
1886                    self.state.repair_grants[index].head =
1887                        git::rev_parse(&winner.worktree, "HEAD").await?;
1888                    self.state.repair_grants[index].phase = Phase::Reviewing;
1889                    if grant.receipt.request.kind == Kind::BaseConflict {
1890                        self.state.base_sync_recovery.stop = None;
1891                        if let Some(sync) = &mut self.state.base_sync {
1892                            sync.tip = grant
1893                                .receipt
1894                                .request
1895                                .tip
1896                                .clone()
1897                                .context("repair base tip missing")?;
1898                            sync.conflict = None;
1899                            sync.behind = 0;
1900                        }
1901                        refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1902                    }
1903                    self.state.save()?;
1904                }
1905                Phase::Reviewing => {
1906                    self.review_loop_pass().await?;
1907                    if self.resource_halt()? {
1908                        return Ok(());
1909                    }
1910                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1911                    let clean = self.state.reviews.len() > grant.review_start
1912                        && self.state.reviews.last().is_some_and(|r| {
1913                            r.head == head
1914                                && r.clean
1915                                && r.expected > 0
1916                                && r.answered == r.expected
1917                                && matches!(
1918                                    r.e2e_status(),
1919                                    E2eStatus::Passed | E2eStatus::NotConfigured
1920                                )
1921                        });
1922                    if !clean {
1923                        // A confirmation review carries no additional fix budget.
1924                        self.state.repair_grants[index].phase = if grant.receipt.request.kind
1925                            == Kind::Review
1926                            && grant.spent < grant.receipt.request.rounds
1927                            && self.state.reviews.last().is_some_and(|r| {
1928                                r.fix.is_none()
1929                                    && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1930                            }) {
1931                            Phase::Pending
1932                        } else {
1933                            Phase::Complete
1934                        };
1935                        self.state.status = RunStatus::Blocked;
1936                        self.state.event(
1937                            "repair-grant",
1938                            "fresh review did not clear the repaired head; no publication",
1939                        );
1940                        self.state.save()?;
1941                        if self.state.repair_grants[index].phase == Phase::Pending {
1942                            continue;
1943                        }
1944                        return Ok(());
1945                    }
1946                    self.state.repair_grants[index].phase = Phase::Gating;
1947                    self.state.save()?;
1948                }
1949                Phase::Gating => {
1950                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1951                    self.state.reconcile_verification_head(&head);
1952                    let wait = self.repository_wait();
1953                    if self
1954                        .state
1955                        .reviews
1956                        .last()
1957                        .is_none_or(|r| r.head != head || !r.clean)
1958                    {
1959                        self.state.status = RunStatus::Blocked;
1960                        self.state.repair_grants[index].phase = Phase::Complete;
1961                        self.state.event(
1962                            "repair-grant",
1963                            "head changed since fresh review; repair held",
1964                        );
1965                        self.state.save()?;
1966                        return Ok(());
1967                    }
1968                    if !verify_current_head(&mut self.state, &winner, &wait).await? {
1969                        return Ok(());
1970                    }
1971                    let outcomes = self.run_gate(&winner).await?;
1972                    if verify_inconclusive(&outcomes) {
1973                        return Ok(());
1974                    }
1975                    let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1976                    if after != head {
1977                        self.state.repair_grants[index].review_start = self.state.reviews.len();
1978                        self.state.repair_grants[index].head = after;
1979                        self.state.repair_grants[index].phase = Phase::Reviewing;
1980                        self.state.status = RunStatus::Blocked;
1981                        self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1982                        self.state.save()?;
1983                        return Ok(());
1984                    }
1985                    let passed = outcomes.iter().all(CommandOutcome::ok);
1986                    // Follow-up grants owe the same guarded checkout delivery
1987                    // as the conflict repair that introduced the new head.
1988                    if passed
1989                        && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1990                    {
1991                        return Ok(());
1992                    }
1993                    self.state.repair_grants[index].succeeded = passed;
1994                    self.state.gate = outcomes;
1995                    self.state.gate_ran = true;
1996                    self.state.status = if passed {
1997                        RunStatus::Gating
1998                    } else {
1999                        RunStatus::Blocked
2000                    };
2001                    self.state.repair_grants[index].phase = if !passed
2002                        && grant.receipt.request.kind == Kind::Gate
2003                        && grant.spent < grant.receipt.request.rounds
2004                    {
2005                        Phase::Pending
2006                    } else {
2007                        Phase::Complete
2008                    };
2009                    if passed && self.state.config.merge.mode == MergeMode::Pr {
2010                        // Persist publication debt with the successful checkpoint.
2011                        self.state.pr_publication_pending = true;
2012                    } else if passed {
2013                        self.state.status = RunStatus::Ready;
2014                    }
2015                    self.state.save()?;
2016                    if !passed {
2017                        continue;
2018                    }
2019                    // Existing PR ownership and push lease stay in place. The
2020                    // normal publisher owes the new reviewed/gated SHA, and
2021                    // land's fresh owner approval remains mandatory.
2022                    if self.state.config.merge.mode != MergeMode::Pr {
2023                        self.state.status = RunStatus::Ready;
2024                        self.state.event(
2025                            "repair-grant",
2026                            "repair verified locally; grant carries no merge authorization",
2027                        );
2028                        self.state.save()?;
2029                        return Ok(());
2030                    }
2031                    return self.merge().await;
2032                }
2033                Phase::Complete => return Ok(()),
2034            }
2035        }
2036    }
2037
2038    async fn execute_graph(&mut self) -> Result<()> {
2039        // Even a terminal run may reenter through an explicit recovery action.
2040        // Restore before prep shortcuts, saved conversations or graph nodes use
2041        // a disposable path; GC may already have removed its original objects.
2042        let archive_home = crate::run::home().canonicalize()?;
2043        if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2044            let repo = self.state.repo.clone();
2045            let _coordination = self.coordinate(&repo).await?;
2046            ensure_setup_config(&mut self.state, &repo).await?;
2047            crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
2048        }
2049        self.state.configure_recovery_resume()?;
2050        if self
2051            .state
2052            .repair_grants
2053            .last()
2054            .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2055        {
2056            return self.execute_granted_repair().await;
2057        }
2058        if self
2059            .state
2060            .repair_grants
2061            .last()
2062            .is_some_and(|g| !g.succeeded)
2063        {
2064            self.state.status = RunStatus::Blocked;
2065            self.state.save()?;
2066            return Ok(());
2067        }
2068        // Preservation may invalidate an old empty classification. Resolve it
2069        // before any resume shortcut can reuse a verdict or publish a branch.
2070        self.persist_candidates().await?;
2071        // A saved base stop outranks stage markers, including legacy Voting.
2072        // Recovery never reenters implementation, deliberation or voting.
2073        if self.state.base_sync_blocked() {
2074            self.sync_to_base().await?;
2075            if !self.state.base_sync_blocked()
2076                && self.state.status != RunStatus::AlreadyInBase
2077                && !self.resource_halt()?
2078            {
2079                self.finish_after_tally().await?;
2080            }
2081            return Ok(());
2082        }
2083        if self
2084            .state
2085            .merge_decision
2086            .as_ref()
2087            .is_some_and(|decision| !decision.closed)
2088            && self.state.status != RunStatus::Landing
2089        {
2090            if crate::merge_decision::resumable_hold(&self.state) {
2091                self.state.merge = Some(MergeOutcome {
2092                    mode: MergeMode::Pr,
2093                    ok: false,
2094                    detail: self
2095                        .state
2096                        .merge_decision
2097                        .as_ref()
2098                        .unwrap()
2099                        .target
2100                        .pr
2101                        .clone(),
2102                    empty: false,
2103                });
2104                self.run_land().await?;
2105                self.settle_questions();
2106                return Ok(());
2107            }
2108            bail!(
2109                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2110            );
2111        }
2112        if self.state.config.graph.selected_implementer.is_some()
2113            && self.state.status == RunStatus::Blocked
2114            && self.state.tally.is_none()
2115            && self.state.candidate_preservation.is_empty()
2116            && self.state.interrupted_implementations.is_empty()
2117        {
2118            bail!(
2119                "selected implementation is held; explicitly release the owning task to retry it"
2120            );
2121        }
2122
2123        if self.state.merge_repairs.last().is_some_and(|r| {
2124            matches!(
2125                r.phase,
2126                crate::run::MergeRepairPhase::Requested
2127                    | crate::run::MergeRepairPhase::Fixing
2128                    | crate::run::MergeRepairPhase::Reviewing
2129            )
2130        }) {
2131            if self.repair_merge_review().await? {
2132                self.run_land().await?;
2133            }
2134            self.settle_questions();
2135            return Ok(());
2136        }
2137        // Publication already passed review and gate. Resume only that node:
2138        // syncing to a newer base here could change the verified commit.
2139        if self.state.pr_publication_pending {
2140            self.merge().await?;
2141            if self.resource_halt()? {
2142                return Ok(());
2143            }
2144            self.state.save()?;
2145            self.settle_questions();
2146            return Ok(());
2147        }
2148        // A run that already lost its quorum never resumes into the verdict
2149        // machinery: `deliberate` and `vote` would otherwise clobber the
2150        // stalled marker back to Voting and the run would keep going past a
2151        // verdict that is no longer trustworthy. Everything already recorded is
2152        // kept, so the run stays resumable (or foldable) for a human to pick up.
2153        //
2154        // On --resume the run gets one chance to repair itself: the seats a
2155        // rate limit took out are re-asked. If their quota has since reset and
2156        // the quorum is restored, the run picks up and finishes; otherwise it
2157        // stays stale and still-resumable for a later retry. If it does not
2158        // recover, the returned status stays `Stalled` and nothing was
2159        // clobbered (the recovery only mutates entries for the lost seats).
2160        if self.state.status == RunStatus::Stalled {
2161            if self.recover_stall().await? {
2162                self.finish_after_tally().await?;
2163            } else {
2164                // Still below quorum: persist the marker and stay resumable.
2165                self.state.save()?;
2166            }
2167            return Ok(());
2168        }
2169        // A run parked inside `land` - watching CI, mid fix-round, or
2170        // waiting on the owner's merge approval - resumes directly into it,
2171        // never back through `prep`. Everything before `merge` already
2172        // concluded; that is the only way `status` reaches `Landing` in the
2173        // first place. Re-walking `review_loop` first would also be actively
2174        // wrong: its own status recomputation (see its doc) treats any
2175        // clean round as reason to set `status` to `Gating`, which would
2176        // clobber this marker before `merge` ever ran, and this run would
2177        // never find its way back into `land` at all.
2178        if self.state.status == RunStatus::Landing
2179            || crate::handover::landing_checkpoint(&self.state)
2180        {
2181            self.state.land_sync_pending = true;
2182            self.state.save()?;
2183            self.run_land().await?;
2184            // `run_land` may have settled the run right here - CI came back
2185            // green and the PR merged, say - without ever passing back
2186            // through `merge`'s own trailing call. Whatever it left `status`
2187            // as is what this has to read.
2188            self.settle_questions();
2189            return Ok(());
2190        }
2191        // A run parked at `merge` on the owner's word about a withheld PR text
2192        // resumes straight into `merge`. Re-walking the tail would sync to the
2193        // base again (possibly rebasing, possibly spending the fixer) after the
2194        // gate had passed, and then push a tree nobody reviewed or gated.
2195        if self.state.github_text.is_some()
2196            && self.state.merge.is_none()
2197            && self.state.status == RunStatus::Gating
2198        {
2199            self.merge().await?;
2200            if self.resource_halt()? {
2201                return Ok(());
2202            }
2203            self.state.save()?;
2204            return Ok(());
2205        }
2206        if let Err(e) = self.prep().await {
2207            if e.is::<crate::repo_lock::ResourceBlocked>() {
2208                return Err(e);
2209            }
2210            self.state.status = RunStatus::Blocked;
2211            self.state
2212                .prep_failure
2213                .get_or_insert_with(|| format!("{e:#}"));
2214            self.state
2215                .event("prep", format!("preparation stopped: {e:#}"));
2216            self.state.save()?;
2217            return Err(e);
2218        }
2219        if self.park_here()? {
2220            return Ok(());
2221        }
2222        self.advise().await?;
2223        if self.park_here()? {
2224            return Ok(());
2225        }
2226        self.implement().await?;
2227        if self.park_here()? {
2228            return Ok(());
2229        }
2230        // `after_implement` already saved the state and settled any open
2231        // questions when it set this; nothing later in the graph has
2232        // anything to judge.
2233        if self.state.status == RunStatus::VerifiedNoop {
2234            return Ok(());
2235        }
2236        self.judge().await?;
2237        if self.park_here()? {
2238            return Ok(());
2239        }
2240        self.deliberate().await?;
2241        if self.park_here()? {
2242            return Ok(());
2243        }
2244        self.vote().await?;
2245        if self.park_here()? {
2246            return Ok(());
2247        }
2248        self.tally()?;
2249        // A verdict that lost its quorum is not trustworthy: do not review,
2250        // gate, or merge on it. Everything already done is kept, so the run
2251        // stays resumable (or foldable); the human can replace the agent that
2252        // ran out of quota and pick it up.
2253        if self.state.status == RunStatus::Stalled {
2254            // Persist the stalled marker now — the normal end-of-execute save
2255            // below is below this early return, and without it a resumed run
2256            // would reload a pre-tally status and keep going.
2257            self.state.save()?;
2258            return Ok(());
2259        }
2260        self.finish_after_tally().await?;
2261        Ok(())
2262    }
2263
2264    /// A wave could not start its agents because a shared resource stayed busy
2265    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
2266    /// resumable. Unstarted seats have no result; completed peers retain
2267    /// their saved candidate state.
2268    fn stop_for_resource(&mut self) -> Result<()> {
2269        // Resource acquisition observes cancellation inside a node. Complete
2270        // the same durable park as every other node boundary before returning.
2271        if self.pause.parked() || self.interrupt.parked() {
2272            self.park_here()?;
2273            return Ok(());
2274        }
2275        if self.state.resource_block.is_some() {
2276            self.state.status = RunStatus::Blocked;
2277        }
2278        self.state.save()
2279    }
2280
2281    /// Stop walking the graph if a node ended on a resource block.
2282    fn resource_halt(&mut self) -> Result<bool> {
2283        if self.state.resource_block.is_none() {
2284            return Ok(false);
2285        }
2286        self.stop_for_resource()?;
2287        Ok(true)
2288    }
2289
2290    /// Park here if asked to, recording it in the run's own timeline.
2291    ///
2292    /// Returns whether the caller should stop walking the graph. The state is
2293    /// saved either way by the node that just finished; this adds the event so
2294    /// the operator's card says why a run that is neither finished nor moving
2295    /// is sitting where it is.
2296    fn park_here(&mut self) -> Result<bool> {
2297        let parking = self.pause.parked() || self.interrupt.parked();
2298        if !parking && self.state.resource_block.is_some() {
2299            self.stop_for_resource()?;
2300            return Ok(true);
2301        }
2302        // Either handle asking is enough - see `Pause`'s own doc for why
2303        // they are never the same one. `interrupt` is checked second so a
2304        // reason it carries is preferred in the message below over a plain
2305        // shutdown park racing it at the same boundary.
2306        if !self.pause.parked() && !self.interrupt.parked() {
2307            return Ok(false);
2308        }
2309        if self.state.parked {
2310            return Ok(true);
2311        }
2312        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2313        if let Some(block) = &mut self.state.resource_block
2314            && matches!(
2315                block.cause,
2316                crate::run::ResourceBlockCause::Cancelled { .. }
2317            )
2318        {
2319            block.cause = crate::run::ResourceBlockCause::Cancelled {
2320                park: cause.clone(),
2321            };
2322        }
2323        let why = match cause.as_ref().map(ParkCause::label) {
2324            Some(reason) => format!(
2325                "parked after `{}` ({reason}) — resume to carry on from here",
2326                self.state.status.as_str()
2327            ),
2328            None => format!(
2329                "parked after `{}` — resume to carry on from here",
2330                self.state.status.as_str()
2331            ),
2332        };
2333        self.state.event("park", why);
2334        self.state
2335            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2336        self.state.parked = true;
2337        self.state.save()?;
2338        Ok(true)
2339    }
2340
2341    /// Only a new explicit operator approval may retry failed preparation.
2342    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2343        if self.state.prep_failure.is_none() {
2344            return Ok(());
2345        }
2346        let failed_at = self
2347            .state
2348            .events
2349            .iter()
2350            .rev()
2351            .find(|event| event.node == "prep")
2352            .map_or(self.state.updated_at, |event| event.at);
2353        anyhow::ensure!(
2354            approval_at > failed_at,
2355            "preparation retry requires a new explicit release after the failure"
2356        );
2357        self.state.prep_failure = None;
2358        self.state.prep_complete = Some(false);
2359        self.state
2360            .event("prep-retry", "operator approved retry of preparation");
2361        Ok(())
2362    }
2363
2364    /// Hand the runner the pause `magi serve`'s own shutdown watches.
2365    pub fn on_pause(&mut self, pause: Pause) {
2366        self.pause = pause;
2367    }
2368
2369    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
2370    /// scheduler asking this one run - and no other - to park so a task
2371    /// marked [`crate::queue::Task::interrupt`] can run alone. See
2372    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
2373    /// handle.
2374    pub fn watch_interrupt(&mut self, pause: Pause) {
2375        self.interrupt = pause;
2376    }
2377
2378    /// Abandon this run's own open questions, once `status` has actually
2379    /// settled rather than merely paused.
2380    ///
2381    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
2382    /// either back up with the candidates, the review round and the seat
2383    /// sessions already on disk, so a question an implementer asked mid-round
2384    /// may still get a real answer read by a real resume. Only the statuses
2385    /// `resumable` excludes are actually final: the run merged, it reached
2386    /// `Ready` with nothing left to do, it failed outright with no
2387    /// established point to continue from, or every candidate agreed, with
2388    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
2389    /// every one of those the seat that asked is gone for good, exactly like
2390    /// the run being deleted under `magi run rm` - so the same cleanup
2391    /// applies, worded for what actually happened instead of "the run was
2392    /// deleted".
2393    ///
2394    /// Best-effort and silent on success: called from every place `status`
2395    /// can land on one of those three, including ones a resumed run revisits,
2396    /// so it must cost nothing when there was nothing open to begin with.
2397    fn settle_questions(&mut self) {
2398        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2399            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2400        }
2401    }
2402
2403    /// The tail of the graph after a trustworthy tally: fold losers, review,
2404    /// gate, merge, and persist.
2405    fn finish_after_tally(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
2406        Box::pin(self.finish_after_tally_inner())
2407    }
2408
2409    async fn finish_after_tally_inner(&mut self) -> Result<()> {
2410        // Before the losers are folded: their branches are what the graft
2411        // reads from.
2412        self.graft().await?;
2413        if self.resource_halt()? {
2414            return Ok(());
2415        }
2416        self.fold_losers().await?;
2417        // Before review starts, and again right before the gate: a run's
2418        // review rounds can themselves take long enough for the base to move
2419        // a second time, and the gate is the one node whose "green" gets
2420        // acted on.
2421        self.sync_to_base().await?;
2422        if self.hold_base_sync().await? || self.resource_halt()? {
2423            return Ok(());
2424        }
2425        if self.state.status == RunStatus::AlreadyInBase {
2426            return Ok(());
2427        }
2428        self.review_loop().await?;
2429        if self.resource_halt()? {
2430            return Ok(());
2431        }
2432        self.sync_to_base().await?;
2433        if self.hold_base_sync().await? || self.resource_halt()? {
2434            return Ok(());
2435        }
2436        if self.state.status == RunStatus::AlreadyInBase {
2437            return Ok(());
2438        }
2439        self.gate().await?;
2440        self.merge().await?;
2441        if self.resource_halt()? {
2442            return Ok(());
2443        }
2444        self.state.save()?;
2445        Ok(())
2446    }
2447
2448    // ---------------------------------------------------------------- prep
2449
2450    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2451    /// run with the step and its output in the event; it is never swallowed.
2452    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2453    /// resume starts over instead of finding candidates and missing seats.
2454    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2455        let result = match ensure_setup_config(&mut self.state, repo).await {
2456            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2457            Err(e) => Err(e),
2458        };
2459        if let Err(e) = result {
2460            git::worktree_remove_clean(repo, wt).await.ok();
2461            if in_prep {
2462                self.rollback_prep(repo).await;
2463            }
2464            self.state.status = RunStatus::Blocked;
2465            self.state.event(
2466                "setup",
2467                format!("worktree setup failed in {}: {e:#}", wt.display()),
2468            );
2469            self.state.save()?;
2470            return Err(e);
2471        }
2472        Ok(())
2473    }
2474
2475    /// Undo everything `prep` created, so a resume re-runs it whole.
2476    async fn rollback_prep(&mut self, repo: &Path) {
2477        let root = self.state.worktree_root();
2478        for c in std::mem::take(&mut self.state.candidates) {
2479            if matches!(
2480                git::worktree_remove_clean(repo, &c.worktree).await,
2481                Ok(true)
2482            ) {
2483                git::branch_delete(repo, &c.branch).await.ok();
2484            } else {
2485                self.state.candidates.push(c);
2486            }
2487        }
2488        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2489        for j in 1..=seats {
2490            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2491                .await
2492                .ok();
2493        }
2494        for k in 1..=self.state.config.graph.advisors {
2495            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2496                .await
2497                .ok();
2498        }
2499    }
2500
2501    async fn prep(&mut self) -> Result<()> {
2502        if let Some(reason) = &self.state.prep_failure {
2503            bail!(
2504                "candidate preparation previously failed: {reason}; correct the source before retrying"
2505            );
2506        }
2507        if self
2508            .state
2509            .prep_complete
2510            .unwrap_or(!self.state.candidates.is_empty())
2511        {
2512            return Ok(());
2513        }
2514        let repo = self.state.repo.clone();
2515        // Includes rollback, config, refs, checkout creation and setup. No
2516        // repository mutation may precede the real shared lock acquisition.
2517        let _coordination = self.coordinate(&repo).await?;
2518        let base = self.state.base_commit.clone();
2519        let plan = self.state.sources.plan(&repo, &base).await?;
2520        self.rollback_prep(&repo).await;
2521        if !self.state.candidates.is_empty() {
2522            bail!(
2523                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2524            );
2525        }
2526        self.state.prep_complete = Some(false);
2527        self.state.status = RunStatus::Prep;
2528        self.state.save()?;
2529        let repo = self.state.repo.clone();
2530        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2531        let root = self.state.worktree_root();
2532        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2533
2534        // The hook is the write-time half of the blindness contract; the
2535        // presentation filter in `blind` is the half that cannot be bypassed.
2536        let hooks_dir = self.state.dir().join("hooks");
2537        if self.state.config.blind.commit_msg_hook {
2538            std::fs::create_dir_all(&hooks_dir)
2539                .with_context(|| format!("create {}", hooks_dir.display()))?;
2540            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2541            let path = hooks_dir.join("commit-msg");
2542            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2543            make_executable(&path)?;
2544            // Ref-counted rather than a plain idempotent set: with more than
2545            // one run able to be in flight in the same repository at once
2546            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2547            // true?" check cannot tell "another run of mine still needs
2548            // this" from "nobody does", and the run that happens to finish
2549            // first would disable the hook out from under a sibling still
2550            // relying on it.
2551            git::acquire_worktree_config(&repo).await?;
2552            self.state.enabled_worktree_config = true;
2553        }
2554
2555        // `[worktree] setup` hides its products per worktree, which needs the
2556        // same per-worktree config the hook does. Held until fold, like it.
2557        ensure_setup_config(&mut self.state, &repo).await?;
2558
2559        for (index, (spec, label)) in self
2560            .roles
2561            .implementers
2562            .clone()
2563            .into_iter()
2564            .zip(labels)
2565            .enumerate()
2566        {
2567            let branch = self.state.branch_for(label);
2568            let worktree = root.join(format!("cand-{label}"));
2569            // Persist intent before Git can leave a partially created tree.
2570            // Candidate registration follows setup, so it cannot inventory
2571            // crashes or refusals during creation itself.
2572            self.state.event(
2573                "prep-intent",
2574                serde_json::json!({
2575                    "branch": branch,
2576                    "worktree": worktree,
2577                    "start": start,
2578                    "picks": plan.picks,
2579                })
2580                .to_string(),
2581            );
2582            self.state.save()?;
2583            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2584            if self.state.config.blind.commit_msg_hook {
2585                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2586            }
2587            git::local_exclude(&worktree, "/.magi/").await?;
2588            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2589                // Nothing of this prep may survive: a resume must start over,
2590                // not find half the candidates and skip the rest.
2591                git::worktree_remove_clean(&repo, &worktree).await.ok();
2592                git::branch_delete(&repo, &branch).await.ok();
2593                self.rollback_prep(&repo).await;
2594                self.state.status = RunStatus::Blocked;
2595                self.state
2596                    .event("prep", format!("worktree setup failed: {e:#}"));
2597                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2598                self.state.save()?;
2599                return Err(e);
2600            }
2601            self.state.candidates.push(Candidate {
2602                index,
2603                label,
2604                agent: spec.id.clone(),
2605                branch,
2606                worktree,
2607                summary: String::new(),
2608                stat: String::new(),
2609                files: 0,
2610                commits: 0,
2611                empty: false,
2612                failed: None,
2613                verified_noop: None,
2614                duration_ms: 0,
2615                folded: false,
2616            });
2617            self.state.save()?;
2618            for pick in &plan.picks {
2619                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2620                {
2621                    self.state.status = RunStatus::Blocked;
2622                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2623                    self.state
2624                        .event("prep", format!("cannot apply referenced commit: {e}"));
2625                    self.state.save()?;
2626                    return Err(e);
2627                }
2628            }
2629        }
2630
2631        for j in 1..=self.roles.judges.len() {
2632            let wt = root.join(format!("judge-{j}"));
2633            if !wt.exists() {
2634                git::worktree_add_detached(&repo, &wt, &base).await?;
2635                self.setup_seat_worktree(&repo, &wt, true).await?;
2636            }
2637        }
2638
2639        // Disposable, detached checkouts for the design-deliberation stage's
2640        // advisor seats — the same shape as the judges' above, at the same
2641        // base commit, since advisors also only ever read. Sized off the
2642        // configured count directly rather than a resolved roster: unlike
2643        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2644        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2645        // `prep` has no `ResolvedRoles` field to read a count from here.
2646        if self.state.config.graph.advise {
2647            for k in 1..=self.state.config.graph.advisors {
2648                let wt = root.join(format!("advisor-{k}"));
2649                if !wt.exists() {
2650                    git::worktree_add_detached(&repo, &wt, &base).await?;
2651                    self.setup_seat_worktree(&repo, &wt, true).await?;
2652                }
2653            }
2654        }
2655
2656        // A judge cannot tell it is looking at its own patch — the seats keep
2657        // separate conversations — but a panel that shares agents with the
2658        // field is less independent than it looks, and that is worth saying out
2659        // loud once per run rather than leaving it in the config.
2660        let authors: Vec<&str> = self
2661            .roles
2662            .implementers
2663            .iter()
2664            .map(|a| a.id.as_str())
2665            .collect();
2666        let overlap: Vec<String> = self
2667            .roles
2668            .judges
2669            .iter()
2670            .enumerate()
2671            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2672            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2673            .collect();
2674        if !overlap.is_empty() {
2675            let note = format!(
2676                "{} also authored a candidate; blind, but the panel is less \
2677                 independent than {} distinct agents would be",
2678                overlap.join(", "),
2679                self.roles.judges.len()
2680            );
2681            self.state.event("prep", note);
2682        }
2683
2684        self.state.event(
2685            "prep",
2686            format!(
2687                "{} candidates, {} judges, base {} ({})",
2688                self.state.candidates.len(),
2689                self.roles.judges.len(),
2690                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2691                self.state.base_branch
2692            ),
2693        );
2694        self.state.prep_complete = Some(true);
2695        self.state.status = RunStatus::Implementing;
2696        self.state.save()?;
2697        Ok(())
2698    }
2699
2700    // -------------------------------------------------------------- advise
2701
2702    /// The design-deliberation stage: independent, read-only advisor seats
2703    /// each sketch a design before any implementer touches the repository,
2704    /// and (when at least one produced a usable proposal) a synthesis seat
2705    /// blends them into a brief `implement` carries in every candidate's
2706    /// prompt.
2707    ///
2708    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2709    /// advisors` is the proposal count. Everything here is best-effort and
2710    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2711    /// that cannot reach quota, or a synthesis seat that produced nothing
2712    /// usable all leave `implement` exactly as it was before this stage
2713    /// existed — the task instruction alone — rather than failing the whole
2714    /// competition over an enrichment stage. Every outcome is still recorded
2715    /// as an event, so a run that got nothing from this stage says why.
2716    ///
2717    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2718    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2719    /// resumed run whose stage failed would re-run it, and re-spend the
2720    /// agent calls, on every reentry before `implement`.
2721    ///
2722    /// Also skipped once any candidate shows implementation progress — the
2723    /// exact predicate `implement` itself uses to decide a candidate is no
2724    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2725    /// is not enough: a run created by an older binary that predates this
2726    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2727    /// an already-`Implementing`-or-later run under this build would
2728    /// otherwise walk straight back through `prep` (a no-op once candidates
2729    /// exist) into this node and spawn every advisor seat against worktrees
2730    /// `prep` never recreated — after implementation has already started,
2731    /// which is exactly the invariant this stage exists to guarantee.
2732    async fn advise(&mut self) -> Result<()> {
2733        let implement_untouched = self
2734            .state
2735            .candidates
2736            .iter()
2737            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2738        if !self.state.config.graph.advise || self.state.advise_attempted {
2739            return Ok(());
2740        }
2741        if !implement_untouched {
2742            self.state.event(
2743                "advise",
2744                "skipping the design-deliberation stage: at least one \
2745                 candidate already shows implementation progress, so this \
2746                 run is past the point the stage exists to run before"
2747                    .to_owned(),
2748            );
2749            self.state.advise_attempted = true;
2750            self.state.save()?;
2751            return Ok(());
2752        }
2753        let run_id = self.state.id.clone();
2754        let prompts = self.state.config.prompts.clone();
2755        let instruction = self.state.instruction.clone();
2756        let language = self.state.config.graph.language.clone();
2757        let root = self.state.worktree_root();
2758        let n = self.state.config.graph.advisors;
2759        let where_recorded = self.state.dir().join("run.json");
2760
2761        let seats = match self.state.config.advisors() {
2762            Ok(seats) if !seats.is_empty() => seats,
2763            Ok(_) => {
2764                self.state.event(
2765                    "advise",
2766                    format!(
2767                        "[graph] advisors is 0; skipping the design-deliberation \
2768                         stage and continuing without a synthesis brief (see {})",
2769                        where_recorded.display()
2770                    ),
2771                );
2772                self.state.advise_attempted = true;
2773                self.state.save()?;
2774                return Ok(());
2775            }
2776            Err(e) => {
2777                self.state.event(
2778                    "advise",
2779                    format!(
2780                        "could not resolve advisor seats ({e:#}); continuing \
2781                         without a design-deliberation brief (see {})",
2782                        where_recorded.display()
2783                    ),
2784                );
2785                self.state.advise_attempted = true;
2786                self.state.save()?;
2787                return Ok(());
2788            }
2789        };
2790
2791        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2792        let artifacts = agent::artifacts_dir(&self.state.dir());
2793        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2794
2795        let mut jobs = Vec::new();
2796        for (i, spec) in seats.iter().cloned().enumerate() {
2797            let seat_key = format!("advisor-{}", i + 1);
2798            let seat = self.seat(&seat_key, &spec.id);
2799            jobs.push(SeatJob {
2800                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2801                spec,
2802                seat,
2803                cwd: worktrees[i % worktrees.len()].clone(),
2804                timeout,
2805                allow_write: false,
2806                sessions: false,
2807                artifacts: artifacts.clone(),
2808                stem: seat_key,
2809                handover: None,
2810            });
2811        }
2812
2813        self.state.event(
2814            "advise",
2815            format!(
2816                "{} advisor seat(s) sketching a design in parallel",
2817                jobs.len()
2818            ),
2819        );
2820        let mut quota_losses = Vec::new();
2821        let cache = self.state.config.cache_dir();
2822        let ctx = WaveCtx {
2823            carry_seats: false,
2824            pauses: Some((&self.pause, &self.interrupt)),
2825            run: &run_id,
2826            node: "advise",
2827            prompts: &prompts,
2828            cache: cache.as_deref(),
2829            round: None,
2830        };
2831        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2832        let results = ask_json_wave::<Proposal>(
2833            jobs,
2834            Arc::clone(&self.sem),
2835            self.state.config.graph.retries,
2836            &advisor_roster,
2837            &ctx,
2838            &mut quota_losses,
2839            &mut self.state,
2840            &|p: &Proposal| p.validate(),
2841        )
2842        .await;
2843        if self.state.resource_block.is_some() {
2844            return self.stop_for_resource();
2845        }
2846        self.state.quota.extend(quota_losses);
2847
2848        let mut records = Vec::with_capacity(results.len());
2849        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2850            let agent_id = seat.agent.clone();
2851            self.state.seats.insert(seat.key.clone(), seat);
2852            match res {
2853                Ok((proposal, out)) => {
2854                    self.state
2855                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2856                    records.push(advise::AdvisorRecord::proposed(
2857                        i + 1,
2858                        agent_id,
2859                        proposal,
2860                        out.duration_ms,
2861                    ));
2862                }
2863                Err(e) => {
2864                    self.state.event(
2865                        "advise",
2866                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2867                    );
2868                    records.push(advise::AdvisorRecord::failed(
2869                        i + 1,
2870                        agent_id,
2871                        e.to_string(),
2872                    ));
2873                }
2874            }
2875        }
2876
2877        let mut advice = advise::Advice {
2878            records,
2879            synthesis: None,
2880        };
2881        if advice.proposals().is_empty() {
2882            self.state.event(
2883                "advise",
2884                "no advisor produced a usable proposal; continuing without a \
2885                 synthesis brief"
2886                    .to_owned(),
2887            );
2888        } else {
2889            match self
2890                .synthesize_brief(
2891                    &advice,
2892                    &instruction,
2893                    &language,
2894                    &worktrees[0],
2895                    &artifacts,
2896                    &run_id,
2897                    &prompts,
2898                    cache.as_deref(),
2899                )
2900                .await
2901            {
2902                Ok(Some(text)) => {
2903                    self.state.event(
2904                        "advise",
2905                        "synthesized a design brief for the implementer".to_owned(),
2906                    );
2907                    advice.synthesis = Some(text);
2908                }
2909                Ok(None) => {
2910                    self.state.event(
2911                        "advise",
2912                        "the synthesis seat produced nothing usable; continuing \
2913                         without a design brief"
2914                            .to_owned(),
2915                    );
2916                }
2917                Err(e) => {
2918                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2919                        return Err(e);
2920                    }
2921                    self.state.event(
2922                        "advise",
2923                        format!("could not synthesize a design brief: {e:#}"),
2924                    );
2925                }
2926            }
2927        }
2928        advise::apply_reflection(&mut advice);
2929
2930        self.state.advice = Some(advice);
2931        self.state.advise_attempted = true;
2932        self.state.save()?;
2933        Ok(())
2934    }
2935
2936    /// The synthesis seat: reads every advisor's proposal and blends them
2937    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2938    /// out of [`Runner::advise`] only for readability — it is not called
2939    /// anywhere else.
2940    ///
2941    /// Picked the same way [`crate::talk`]'s standing conversation and
2942    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2943    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2944    /// order (a claude seat, else the first runnable agent in roster order)
2945    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2946    /// in [`crate::config`] for why a dedicated field exists here at all.
2947    #[allow(clippy::too_many_arguments)]
2948    async fn synthesize_brief(
2949        &mut self,
2950        advice: &advise::Advice,
2951        instruction: &str,
2952        language: &str,
2953        cwd: &Path,
2954        artifacts: &Path,
2955        run_id: &str,
2956        prompts: &Prompts,
2957        cache: Option<&Path>,
2958    ) -> Result<Option<String>> {
2959        let chain = agent::pick_chain(
2960            &self.state.config.agents,
2961            self.state.config.roles.synthesizer.as_ref(),
2962            &agent::installed,
2963            "synthesizer",
2964        )?;
2965        let proposals = advice.proposals();
2966        let mut prompt = prompt::with_overlay(
2967            prompt::synthesize_brief(instruction, &proposals, language),
2968            prompts.overlay("advise"),
2969        );
2970        if cache.is_some() {
2971            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2972            // below — see `prompt::build_cache_note`'s doc for why telling a
2973            // read-only seat to build through the shared cache is exactly how
2974            // a sandbox's write refusal gets misread as a defect.
2975            prompt.push('\n');
2976            prompt.push_str(&prompt::build_cache_note("advise", false));
2977        }
2978        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2979        // Each id is tried once, in order; a quota hit, error or unusable
2980        // answer moves to the next. The seat is single-turn (`sessions:
2981        // false`) and the prompt is the whole context, so a fallback agent
2982        // needs nothing carried over.
2983        let mut last = None;
2984        for (n, spec) in chain.iter().enumerate() {
2985            if n > 0 {
2986                self.state
2987                    .event("advise", format!("synthesis falling back to {}", spec.id));
2988            }
2989            let mut seat = self.seat("advise-synthesis", &spec.id);
2990            let wait = self.repository_wait();
2991            let outcome = wait
2992                .scope(agent::invoke(
2993                    spec,
2994                    &mut seat,
2995                    &Invocation {
2996                        coordination: self.state.config.coordination.as_ref(),
2997                        fetched_base: None,
2998                        cwd,
2999                        prompt: &prompt,
3000                        timeout,
3001                        allow_write: false,
3002                        sessions: false,
3003                        artifacts,
3004                        stem: &if n == 0 {
3005                            "advise-synthesis".to_owned()
3006                        } else {
3007                            format!("advise-synthesis-{}", spec.id)
3008                        },
3009                        run: run_id,
3010                        node: "advise",
3011                        cache_dir: None,
3012                        attachments: &[],
3013                        writable: &[],
3014                    },
3015                ))
3016                .await;
3017            if outcome.is_ok() {
3018                self.state.seats.insert(seat.key.clone(), seat);
3019            }
3020            let advance = agent::chain_advances(&outcome);
3021            last = Some(outcome);
3022            if !advance {
3023                break;
3024            }
3025        }
3026        // Exhausted: the last attempt's result is what a single failed seat
3027        // would have produced.
3028        let out = last.expect("a chain holds at least one agent")?;
3029        if !out.usable() {
3030            return Ok(None);
3031        }
3032        let text =
3033            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3034        Ok((!text.trim().is_empty()).then_some(text))
3035    }
3036
3037    // ----------------------------------------------------------- implement
3038
3039    async fn implement(&mut self) -> Result<()> {
3040        self.persist_candidates().await?;
3041        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3042        // agent files with `magi task add` name the run that paid for it. The
3043        // prompt overlay is cloned alongside it because the waves borrow it
3044        // while `self` is mutably borrowed by the node's own bookkeeping.
3045        let run_id = self.state.id.clone();
3046        let prompts = self.state.config.prompts.clone();
3047        let todo: Vec<usize> = self
3048            .state
3049            .candidates
3050            .iter()
3051            .enumerate()
3052            .filter(|(_, c)| {
3053                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3054                    && c.failed.is_none()
3055                    && !c.empty
3056            })
3057            .map(|(i, _)| i)
3058            .collect();
3059        if todo.is_empty() {
3060            return self.after_implement();
3061        }
3062        self.state.status = RunStatus::Implementing;
3063
3064        let language = self.state.config.graph.language.clone();
3065        let github_language = self.state.config.graph.github_language.clone();
3066        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3067        let sessions = self.state.config.graph.sessions;
3068        let artifacts = agent::artifacts_dir(&self.state.dir());
3069        // The design-deliberation stage's blended brief, when `advise` found
3070        // one — carried into every implementer's prompt the same way
3071        // regardless of which candidate it is.
3072        let brief = self
3073            .state
3074            .advice
3075            .as_ref()
3076            .and_then(|a| a.synthesis.as_deref())
3077            .map(str::to_owned);
3078        let attachments = self.state.attachments.clone();
3079
3080        let mut jobs = Vec::new();
3081        for &i in &todo {
3082            let (index, label, worktree) = {
3083                let c = &self.state.candidates[i];
3084                (c.index, c.label, c.worktree.clone())
3085            };
3086            let spec = self.roles.implementers[index].clone();
3087            let seat_key = format!("impl-{label}");
3088            let seat = self.seat(&seat_key, &spec.id);
3089            let instruction = seeded_instruction(&self.state);
3090            jobs.push(SeatJob {
3091                spec,
3092                seat,
3093                prompt: prompt::implement_in(
3094                    &instruction,
3095                    &worktree.to_string_lossy(),
3096                    &language,
3097                    &github_language,
3098                    brief.as_deref(),
3099                    &attachments,
3100                ),
3101                cwd: worktree,
3102                timeout,
3103                allow_write: true,
3104                sessions,
3105                artifacts: artifacts.clone(),
3106                stem: format!("impl-{label}"),
3107                handover: None,
3108            });
3109        }
3110
3111        self.state.event(
3112            "implement",
3113            format!("{} candidates in parallel", jobs.len()),
3114        );
3115        // Kept so a seat whose CLI hung up can be asked again from the same
3116        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
3117        // can update a seat's own entry once a fallback agent takes it over —
3118        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
3119        // whichever agent actually answered, not the one that quota'd out.
3120        let mut sent = jobs.clone();
3121        let cache = self.state.config.cache_dir();
3122        let ctx = WaveCtx {
3123            carry_seats: false,
3124            pauses: Some((&self.pause, &self.interrupt)),
3125            run: &run_id,
3126            node: "implement",
3127            prompts: &prompts,
3128            cache: cache.as_deref(),
3129            round: None,
3130        };
3131        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3132        if !wave_blocked(&results) {
3133            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3134                .await;
3135            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3136                .await?;
3137            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3138                .await;
3139        }
3140        // Do not finalize failed candidates when the entire wave needs machine
3141        // recovery: leaving them pending resumes this node with its work intact.
3142        if !results.is_empty()
3143            && results
3144                .iter()
3145                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3146        {
3147            let losses: Vec<_> = results
3148                .iter()
3149                .map(|(_, seat, out)| {
3150                    let AgentOutcome::Quota(o) = out else {
3151                        unreachable!()
3152                    };
3153                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3154                })
3155                .collect();
3156            self.state.quota.extend(losses.clone());
3157            for (_, seat, _) in results {
3158                self.state.seats.insert(seat.key.clone(), seat);
3159            }
3160            for &i in &todo {
3161                let label = self.state.candidates[i].label;
3162                self.state.interrupted_implementations.insert(label);
3163                self.state
3164                    .candidate_preservation
3165                    .insert(label, "machine outage awaiting persistence".into());
3166            }
3167            // Preserve partial edits too, while keeping every interrupted seat
3168            // pending even when its tree already contains an inherited patch.
3169            block_machine_outage(&mut self.state, "implement", &losses);
3170            self.state.save()?;
3171            self.persist_candidates().await?;
3172            for &i in &todo {
3173                self.state
3174                    .interrupted_implementations
3175                    .insert(self.state.candidates[i].label);
3176            }
3177            return self.stop_for_resource();
3178        }
3179        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3180
3181        let selected_failed = self.state.config.graph.selected_implementer.is_some()
3182            && results
3183                .iter()
3184                .any(|(_, _, out)| FailClass::of(out).is_some());
3185        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3186            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3187                // Executed turns interrupted by disk pressure already carry
3188                // a preservation barrier from wave. Admission-only waits do
3189                // not. Keep the interrupted seat's session for recovery too.
3190                if self
3191                    .state
3192                    .candidate_preservation
3193                    .contains_key(&self.state.candidates[i].label)
3194                {
3195                    self.state.seats.insert(seat.key.clone(), seat);
3196                }
3197                continue;
3198            }
3199            let seat_key = seat.key.clone();
3200            // A quota fallback (`resume_seat_handovers`) may have handed this
3201            // seat to a different agent than the one `prep` recorded on the
3202            // candidate; the stats tables and any later fixer-defaults-to-
3203            // winner's-author lookup must credit whoever actually answered —
3204            // unless every fallback also quota'd out, in which case nobody
3205            // actually answered and crediting the last agent tried would
3206            // erase every earlier agent's own quota loss from the stats
3207            // tables instead of just this one seat's.
3208            let agent = seat.agent.clone();
3209            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3210            self.state.seats.insert(seat.key.clone(), seat);
3211            let label = self.state.candidates[i].label;
3212            self.state.interrupted_implementations.remove(&label);
3213
3214            let (summary, duration, failed, verified_claim) = match out {
3215                AgentOutcome::Ok(o) => {
3216                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3217                    let failed = (!o.usable()).then(|| {
3218                        if o.timed_out {
3219                            "agent timed out".to_owned()
3220                        } else {
3221                            format!(
3222                                "agent exited with {}",
3223                                crate::notices::exit_status(o.exit_code)
3224                            )
3225                        }
3226                    });
3227                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3228                    (text, o.duration_ms, failed, verified_claim)
3229                }
3230                // Left un-resumed by `resume_undelivered` (a dirty tree
3231                // already rescues the work, or there was no session left to
3232                // resume into) — reported like the ordinary failure it is,
3233                // never as if `o.text` (the CLI's raw error JSON) were an
3234                // answer.
3235                AgentOutcome::Dropped(o) => {
3236                    let why = o
3237                        .dropped
3238                        .as_ref()
3239                        .map(|d| d.why.as_str())
3240                        .unwrap_or("the CLI ended the stream without delivering its answer");
3241                    (
3242                        String::new(),
3243                        o.duration_ms,
3244                        Some(format!("the CLI dropped the stream ({why})")),
3245                        None,
3246                    )
3247                }
3248                AgentOutcome::Quota(o) => {
3249                    self.state.quota.push(QuotaLoss::from_report(
3250                        seat_key,
3251                        "implement",
3252                        o.quota.as_ref(),
3253                    ));
3254                    (
3255                        String::new(),
3256                        o.duration_ms,
3257                        Some("rate limited (quota); produced no change".to_owned()),
3258                        None,
3259                    )
3260                }
3261                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3262                    (String::new(), 0, Some(e), None)
3263                }
3264            };
3265
3266            // Save every completed turn before any fallible Git operation. A failed
3267            // rescue must not erase the reply or prevent collection of other seats.
3268            let c = &mut self.state.candidates[i];
3269            if !exhausted_the_fallback_chain {
3270                c.agent = agent;
3271            }
3272            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3273            c.duration_ms = duration;
3274            c.failed = failed;
3275            c.verified_noop = verified_claim;
3276            self.state
3277                .candidate_preservation
3278                .insert(label, "awaiting persistence".to_owned());
3279        }
3280        self.state.save()?;
3281        self.persist_candidates().await?;
3282
3283        if resource_blocked {
3284            return self.stop_for_resource();
3285        }
3286        if selected_failed {
3287            self.state.status = RunStatus::Blocked;
3288            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3289            self.state.save()?;
3290            bail!(
3291                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3292            );
3293        }
3294        self.after_implement()
3295    }
3296
3297    /// Retry only persistence after a completed implementation turn.
3298    async fn persist_candidates(&mut self) -> Result<()> {
3299        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3300        for label in labels {
3301            let i = self
3302                .state
3303                .candidates
3304                .iter()
3305                .position(|c| c.label == label)
3306                .context("pending candidate is missing")?;
3307            let c = self.state.candidates[i].clone();
3308            let interrupted = self.state.interrupted_implementations.contains(&label);
3309            let result = async {
3310                let wait = crate::repo_lock::WaitPolicy::new(
3311                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3312                    || false,
3313                );
3314                let _coordination =
3315                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3316                // Best effort evidence requires no index writes. A successful snapshot
3317                // never authorizes removal of a tree whose rescue failed.
3318                if let Err(e) = git::snapshot_uncommitted(
3319                    &c.worktree,
3320                    &self
3321                        .state
3322                        .dir()
3323                        .join("artifacts")
3324                        .join(format!("candidate-{label}-recovery")),
3325                )
3326                .await
3327                {
3328                    self.state.event(
3329                        "preserve",
3330                        format!("candidate {label} snapshot incomplete: {e:#}"),
3331                    );
3332                }
3333                let rescued = git::rescue_commit(
3334                    &c.worktree,
3335                    &format!("magi: candidate {label} (uncommitted work)"),
3336                )
3337                .await?;
3338                self.state.note_withheld("implement", &rescued.withheld);
3339                let base = &self.state.base_commit;
3340                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3341                let patch = git::diff(&c.worktree, base, "HEAD").await?;
3342                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3343                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3344                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3345                if c.empty && commits > 0 && !patch.trim().is_empty()
3346                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
3347                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3348                        || self.state.tally.is_some())
3349                {
3350                    // Keep the old classification and pending marker so every
3351                    // retry stops too. Retain decisions as recovery evidence;
3352                    // only a supported recovery operation may replace them.
3353                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3354                }
3355                let c = &mut self.state.candidates[i];
3356                c.commits = commits;
3357                c.stat = stat;
3358                c.files = files;
3359                let no_changes = commits == 0 || patch.trim().is_empty();
3360                c.empty = no_changes && !interrupted;
3361                if interrupted && no_changes {
3362                    c.failed = None;
3363                    c.verified_noop = None;
3364                }
3365                if !c.empty {
3366                    c.failed = None;
3367                    c.verified_noop = None;
3368                }
3369                self.state.event(
3370                    "preserve",
3371                    format!("candidate {label}: {files} files, {commits} commits"),
3372                );
3373                Ok::<_, anyhow::Error>(())
3374            }
3375            .await;
3376            match result {
3377                Ok(()) => {
3378                    self.state.candidate_preservation.remove(&label);
3379                    if !interrupted || self.state.candidates[i].files > 0 {
3380                        self.state.interrupted_implementations.remove(&label);
3381                    }
3382                }
3383                Err(e) => {
3384                    let reason = format!("candidate {label} preservation failed: {e:#}");
3385                    self.state.event("preserve", &reason);
3386                    self.state.candidate_preservation.insert(label, reason);
3387                }
3388            }
3389            if self.state.candidate_preservation.is_empty()
3390                && self.state.status == RunStatus::Blocked
3391                && self.state.candidates.iter().all(|c| c.failed.is_none())
3392            {
3393                // Persist the recovered phase with the cleared pending marker;
3394                // a crash between saves must not turn this into a selected-seat
3395                // failure hold on the next resume.
3396                self.state.status = RunStatus::Implementing;
3397                self.state.event(
3398                    "preserve",
3399                    "candidate persistence recovered; implementation can continue",
3400                );
3401            }
3402            self.state.save()?;
3403        }
3404        if !self.state.candidate_preservation.is_empty() {
3405            self.state.status = RunStatus::Blocked;
3406            self.state.save()?;
3407            bail!(
3408                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3409            );
3410        }
3411        Ok(())
3412    }
3413
3414    /// Ask again, once, for work a CLI did and then failed to hand over.
3415    ///
3416    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3417    /// status with an empty response and a usage report showing output tokens,
3418    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3419    /// seven minutes and 14,267 output tokens that arrived as an empty
3420    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3421    ///
3422    /// Two conditions, and both matter:
3423    ///
3424    /// - **Only when the tree is untouched.** Often the agent has already
3425    ///   written its files and only the closing message was lost; the rescue
3426    ///   commit below picks that up and there is nothing to ask for. Re-asking
3427    ///   then would pay for a second implementation of work already on disk.
3428    /// - **Once.** A CLI that drops one stream can drop the next, and this
3429    ///   node is the most expensive in the graph.
3430    ///
3431    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3432    /// dropped reply still carried its `conversation_id`, so the seat is asked
3433    /// to finish what it was doing rather than sent the whole task again. It
3434    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3435    /// node's - for the same reason a re-ranked judge does: restating finished
3436    /// work is not the work.
3437    ///
3438    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3439    /// same way until it resets, while an abandoned conversation is still
3440    /// there to be picked up.
3441    async fn resume_undelivered(
3442        &mut self,
3443        results: &mut [(usize, SeatState, AgentOutcome)],
3444        sent: &[SeatJob],
3445        prompts: &Prompts,
3446        run_id: &str,
3447    ) {
3448        for (wi, seat, out) in results.iter_mut() {
3449            let Some(dropped) = (match &*out {
3450                AgentOutcome::Dropped(o) => o.dropped.clone(),
3451                _ => None,
3452            }) else {
3453                continue;
3454            };
3455            let Some(job) = sent.get(*wi) else { continue };
3456            // Already on disk? Then only the closing message was lost.
3457            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3458                self.state.event(
3459                    "implement",
3460                    format!(
3461                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3462                         work is in the tree",
3463                        seat.key, dropped.output_tokens, dropped.why
3464                    ),
3465                );
3466                continue;
3467            }
3468            // The re-ask only makes sense as a resume: `resume_after_drop`
3469            // says nothing about the task, trusting the seat to still hold it.
3470            // Without a session to resume — sessions disabled, or this CLI's
3471            // drop shape happened not to carry a session id — that prompt
3472            // would open a brand-new conversation with no context at all,
3473            // which is worse than leaving this as the ordinary failure it
3474            // already is.
3475            if !has_context(&job.spec, seat, job.sessions) {
3476                self.state.event(
3477                    "implement",
3478                    format!(
3479                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3480                         is no session left to resume",
3481                        seat.key, dropped.output_tokens, dropped.why
3482                    ),
3483                );
3484                continue;
3485            }
3486            self.state.event(
3487                "implement",
3488                format!(
3489                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3490                     conversation",
3491                    seat.key, dropped.output_tokens, dropped.why
3492                ),
3493            );
3494            let mut retry = job.clone();
3495            retry.seat = seat.clone();
3496            retry.prompt = prompt::resume_after_drop(&dropped.why);
3497            retry.timeout = retry_budget(job.timeout, true);
3498            retry.stem = format!("{}-resume", job.stem);
3499            let cache = self.state.config.cache_dir();
3500            let ctx = WaveCtx {
3501                carry_seats: false,
3502                pauses: Some((&self.pause, &self.interrupt)),
3503                run: run_id,
3504                node: "implement",
3505                prompts,
3506                cache: cache.as_deref(),
3507                round: None,
3508            };
3509            let (resumed_seat, resumed) =
3510                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3511            *seat = resumed_seat;
3512            *out = resumed;
3513        }
3514    }
3515
3516    /// Fall an implement seat through to the next untried agent in the
3517    /// implementer roster when it lost to quota — or, since the handover was
3518    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3519    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3520    /// quota path itself being unchanged — instead of leaving the
3521    /// seat's loss final the moment one agent's account runs dry.
3522    ///
3523    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3524    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3525    /// truncates to the single slot rotation picked, so a solo task whose one
3526    /// implementer hits quota mid-run used to have nothing else to try. This
3527    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3528    /// unrotated roster — which is the only place the *other* candidates in
3529    /// the machine's roster still exist once `implementers` has been cut down
3530    /// to size.
3531    ///
3532    /// Walks forward from just past the seat's own original position in the
3533    /// roster, never wrapping back to the front: a later candidate slot (say
3534    /// `beta`, the roster's second entry) must fall through to the *next*
3535    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3536    /// almost certainly a different candidate's own agent already — and once
3537    /// the roster's tail is exhausted there is nothing left to fall through
3538    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3539    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3540    /// let this retry that id forever. The loop keeps falling through until
3541    /// an attempt lands something other than `Quota` or the roster's tail
3542    /// runs out of untried ids, at which point the seat is left exactly as
3543    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3544    /// `QuotaLoss` recorded, the candidate failed/empty.
3545    ///
3546    /// `sent` is taken mutably and updated with the fallback agent's spec:
3547    /// `resume_unconfirmed_commands`, which runs after this and also reads
3548    /// `sent`, must see whichever agent actually ended up answering the seat
3549    /// — reading the stale, original spec there would check session
3550    /// eligibility against the wrong CLI and could hand a fallback agent's
3551    /// session id to the agent that just lost the seat to quota.
3552    ///
3553    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3554    /// — `self.seat` only reuses state when the agent id is unchanged, so
3555    /// handing it a different id already gets this for free. Reusing the old
3556    /// seat would resume a different CLI's session as if it were a
3557    /// continuation of this one.
3558    ///
3559    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3560    /// a quota loss cuts an agent off mid-turn, so anything already in the
3561    /// tree is unfinished work, not a completed candidate a re-ask would pay
3562    /// for twice. A dirty tree is rescued into a commit first (the same
3563    /// neutral-identity rescue `implement`'s own outcome loop gives every
3564    /// candidate) so the next agent starts clean.
3565    ///
3566    /// The new agent gets the implementer's full prompt and full
3567    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3568    /// it has no session and no context, and is implementing the task from
3569    /// nothing, unlike a resumed drop which is only restating work already
3570    /// done.
3571    ///
3572    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3573    /// `implement` event, never into `self.state.quota` — that is what
3574    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3575    /// go unspent, and a seat that ultimately recovered on its second or
3576    /// third agent is not the stalled panel that check exists to catch. Only
3577    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3578    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3579    /// outcome loop already has — this helper never pushes to it itself.
3580    async fn resume_seat_handovers(
3581        &mut self,
3582        results: &mut [(usize, SeatState, AgentOutcome)],
3583        sent: &mut [SeatJob],
3584        prompts: &Prompts,
3585        run_id: &str,
3586    ) -> Result<()> {
3587        let instruction = seeded_instruction(&self.state);
3588        let language = self.state.config.graph.language.clone();
3589        let github_language = self.state.config.graph.github_language.clone();
3590        let brief = self
3591            .state
3592            .advice
3593            .as_ref()
3594            .and_then(|a| a.synthesis.as_deref())
3595            .map(str::to_owned);
3596        let attachments = self.state.attachments.clone();
3597        for (wi, seat, out) in results.iter_mut() {
3598            // Who holds the other candidate seats of this wave right now
3599            // (earlier handovers already written back to `sent`).
3600            let others: BTreeSet<String> = sent
3601                .iter()
3602                .enumerate()
3603                .filter(|(j, _)| j != wi)
3604                .map(|(_, j)| j.spec.id.clone())
3605                .collect();
3606            let Some(job) = sent.get_mut(*wi) else {
3607                continue;
3608            };
3609            // Where the seat's own original agent sits in the roster — the
3610            // fallback walk starts just past here, never at the front, so a
3611            // later candidate slot's quota loss does not fall back onto an
3612            // earlier slot's own agent.
3613            let start = self
3614                .roles
3615                .implementer_roster
3616                .iter()
3617                .position(|s| s.id == job.spec.id)
3618                .unwrap_or(0);
3619            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3620            let mut fallback_attempt = 0usize;
3621            let mut prev: Option<FailClass> = None;
3622            while let Some(cur) = FailClass::of(&*out) {
3623                if !should_hand_over(prev.as_ref(), &cur) {
3624                    break;
3625                }
3626                let Some(next) =
3627                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3628                        .cloned()
3629                else {
3630                    break;
3631                };
3632                tried.insert(next.id.clone());
3633                fallback_attempt += 1;
3634
3635                let coordination = self.coordinate(&job.cwd).await?;
3636                if let Ok(r) = git::rescue_commit(
3637                    &job.cwd,
3638                    &format!(
3639                        "magi: candidate {} (uncommitted work before {} fallback)",
3640                        seat.key,
3641                        if cur == FailClass::Quota {
3642                            "quota"
3643                        } else {
3644                            "handover"
3645                        }
3646                    ),
3647                )
3648                .await
3649                {
3650                    self.state.note_withheld("implement", &r.withheld);
3651                }
3652                drop(coordination);
3653
3654                record_handover(
3655                    &mut self.state,
3656                    "implement",
3657                    &seat.key,
3658                    &seat.agent,
3659                    &next.id,
3660                    &cur,
3661                    &fail_reason(&*out),
3662                );
3663                prev = Some(cur.clone());
3664
3665                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3666                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3667                // Kept in sync on `sent` itself, not just the local retry: a
3668                // later helper (`resume_unconfirmed_commands`) reads `sent`
3669                // after this one returns and must see whichever agent is now
3670                // occupying the seat, not the one that just quota'd out —
3671                // otherwise it would judge session/continuation eligibility
3672                // by the wrong CLI and could resend a fallback's session id
3673                // to the agent that lost it the seat in the first place.
3674                job.spec = next.clone();
3675                let mut retry = job.clone();
3676                retry.seat = new_seat;
3677                retry.prompt = prompt::implement_in(
3678                    &instruction,
3679                    &job.cwd.to_string_lossy(),
3680                    &language,
3681                    &github_language,
3682                    brief.as_deref(),
3683                    &attachments,
3684                );
3685                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3686                let cache = self.state.config.cache_dir();
3687                let ctx = WaveCtx {
3688                    carry_seats: false,
3689                    pauses: Some((&self.pause, &self.interrupt)),
3690                    run: run_id,
3691                    node: "implement",
3692                    prompts,
3693                    cache: cache.as_deref(),
3694                    round: None,
3695                };
3696                let (fallback_seat, fallback_out) = run_one(
3697                    retry,
3698                    Arc::clone(&self.sem),
3699                    &ctx,
3700                    &mut self.state,
3701                    fallback_attempt,
3702                )
3703                .await;
3704                *seat = fallback_seat;
3705                *out = fallback_out;
3706            }
3707        }
3708        Ok(())
3709    }
3710
3711    /// Ask an implement seat's own CLI to confirm what it started, once, when
3712    /// its reply reported a command whose completion status it never
3713    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3714    /// that does and does not mean.
3715    ///
3716    /// The completion contract this task asks for, extended to `implement`
3717    /// with the same signal `continue_fix_report` reads for the fixer,
3718    /// rather than a keyword search over the reply or a hard requirement on
3719    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3720    /// a candidate's CLI turn ended cleanly while a test run it had started
3721    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3722    /// named in it at all is untouched by this: `commands` is empty, so
3723    /// there is nothing to be unconfirmed.
3724    ///
3725    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3726    /// this is not about recovering edits that might already be on disk, it
3727    /// is about a result the seat itself never vouched for, which resuming
3728    /// asks for regardless of what the tree already holds. Bounded to one
3729    /// attempt for the same reason `resume_undelivered` is — this is the
3730    /// most expensive node in the graph — and a seat that still cannot
3731    /// confirm on that attempt is left as whatever its (possibly still
3732    /// unconfirmed) reply says; this does not invent a new "failed" reason
3733    /// for a candidate that otherwise produced a real, committed change.
3734    async fn resume_unconfirmed_commands(
3735        &mut self,
3736        results: &mut [(usize, SeatState, AgentOutcome)],
3737        sent: &[SeatJob],
3738        prompts: &Prompts,
3739        run_id: &str,
3740    ) {
3741        for (wi, seat, out) in results.iter_mut() {
3742            let AgentOutcome::Ok(o) = &*out else {
3743                continue;
3744            };
3745            if !has_unconfirmed_command(&o.commands) {
3746                continue;
3747            }
3748            let Some(job) = sent.get(*wi) else { continue };
3749            if !has_context(&job.spec, seat, job.sessions) {
3750                self.state.event(
3751                    "implement",
3752                    format!(
3753                        "{}: the reply named a command whose own CLI never confirmed the exit \
3754                         status of, but there is no session left to resume",
3755                        seat.key
3756                    ),
3757                );
3758                continue;
3759            }
3760            self.state.event(
3761                "implement",
3762                format!(
3763                    "{}: the reply named a command whose own CLI never confirmed the exit \
3764                     status of; resuming the conversation",
3765                    seat.key
3766                ),
3767            );
3768            let mut retry = job.clone();
3769            retry.seat = seat.clone();
3770            retry.prompt = prompt::resume_incomplete(
3771                "a command in your last reply had no confirmed exit status",
3772            );
3773            retry.timeout = retry_budget(job.timeout, true);
3774            retry.stem = format!("{}-confirm", job.stem);
3775            let cache = self.state.config.cache_dir();
3776            let ctx = WaveCtx {
3777                carry_seats: false,
3778                pauses: Some((&self.pause, &self.interrupt)),
3779                run: run_id,
3780                node: "implement",
3781                prompts,
3782                cache: cache.as_deref(),
3783                round: None,
3784            };
3785            let (resumed_seat, resumed) =
3786                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3787            *seat = resumed_seat;
3788            *out = resumed;
3789        }
3790    }
3791
3792    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3793    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3794    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3795    /// that motivated this.
3796    ///
3797    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3798    /// own nudge loop already covers for judge/review/vote seats, and not a
3799    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3800    /// implement seats: here the CLI turn genuinely finished while the node's
3801    /// own work — the fixer's account of what it did — had not. Gated purely
3802    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3803    /// reply, never on any wording in it, so a fixer whose valid, first-try
3804    /// `FixReport` happens to mention having waited on a background test is
3805    /// never resumed — the `Ok(report)` branch at the call site returns
3806    /// before this is ever invoked.
3807    ///
3808    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3809    /// attempt ([`retry_budget`]), nothing attempted once the session is
3810    /// gone, and a quota hit ends the loop immediately rather than retrying a
3811    /// rate limit that fails the same way again.
3812    async fn continue_fix_report(
3813        &mut self,
3814        mut seat: SeatState,
3815        parse_err: String,
3816        job: &SeatJob,
3817        prompts: &Prompts,
3818        run_id: &str,
3819        round: usize,
3820    ) -> (
3821        SeatState,
3822        Option<FixReport>,
3823        Option<String>,
3824        ContinuationRecord,
3825    ) {
3826        let mut last_err = parse_err;
3827        let mut cumulative_wait_ms = 0u64;
3828        let mut attempts = 0usize;
3829        loop {
3830            if !has_context(&job.spec, &seat, job.sessions) {
3831                self.state.event(
3832                    "fix",
3833                    format!(
3834                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3835                         session left to resume into"
3836                    ),
3837                );
3838                let outcome = if attempts == 0 {
3839                    ContinuationOutcome::NoSession
3840                } else {
3841                    ContinuationOutcome::Exhausted
3842                };
3843                return (
3844                    seat,
3845                    None,
3846                    Some(format!("unparsable fix report: {last_err}")),
3847                    ContinuationRecord {
3848                        attempts,
3849                        cumulative_wait_ms,
3850                        outcome,
3851                    },
3852                );
3853            }
3854            if attempts >= MAX_FIX_CONTINUATIONS {
3855                self.state.event(
3856                    "fix",
3857                    format!(
3858                        "round {round}: fixer's reply still had no adoption report after \
3859                         {attempts} continuation(s) ({last_err}); giving up"
3860                    ),
3861                );
3862                return (
3863                    seat,
3864                    None,
3865                    Some(format!(
3866                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3867                    )),
3868                    ContinuationRecord {
3869                        attempts,
3870                        cumulative_wait_ms,
3871                        outcome: ContinuationOutcome::Exhausted,
3872                    },
3873                );
3874            }
3875            attempts += 1;
3876            self.state.event(
3877                "fix",
3878                format!(
3879                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3880                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3881                ),
3882            );
3883            let mut retry = job.clone();
3884            retry.seat = seat.clone();
3885            retry.prompt = prompt::resume_incomplete(&last_err);
3886            retry.timeout = retry_budget(job.timeout, true);
3887            retry.stem = format!("{}-continue{attempts}", job.stem);
3888            let cache = self.state.config.cache_dir();
3889            let ctx = WaveCtx {
3890                carry_seats: false,
3891                pauses: Some((&self.pause, &self.interrupt)),
3892                run: run_id,
3893                node: "fix",
3894                prompts,
3895                cache: cache.as_deref(),
3896                round: Some(round),
3897            };
3898            let (resumed_seat, resumed_out) = run_one(
3899                retry,
3900                Arc::clone(&self.sem),
3901                &ctx,
3902                &mut self.state,
3903                attempts,
3904            )
3905            .await;
3906            seat = resumed_seat;
3907            match resumed_out {
3908                AgentOutcome::Ok(o) => {
3909                    cumulative_wait_ms += o.duration_ms;
3910                    match verdict::extract_json::<FixReport>(&o.text) {
3911                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3912                            self.state.event(
3913                                "fix",
3914                                format!(
3915                                    "round {round}: fixer's adoption report recovered after \
3916                                     {attempts} continuation(s)"
3917                                ),
3918                            );
3919                            return (
3920                                seat,
3921                                Some(report),
3922                                None,
3923                                ContinuationRecord {
3924                                    attempts,
3925                                    cumulative_wait_ms,
3926                                    outcome: ContinuationOutcome::Resumed,
3927                                },
3928                            );
3929                        }
3930                        // The report parsed, but this same reply's own
3931                        // CommandEvidence — the identical record `state.jobs`
3932                        // renders — names a command whose CLI never
3933                        // confirmed an exit status. Read together, that is
3934                        // not a resolved answer: keep nudging rather than
3935                        // accept a report standing next to a command the
3936                        // seat's own CLI cannot vouch for.
3937                        Ok(_) => {
3938                            last_err = "the reply parsed, but it reported a command whose own CLI \
3939                                 never confirmed an exit status"
3940                                .to_owned();
3941                        }
3942                        Err(e) => last_err = e.to_string(),
3943                    }
3944                }
3945                AgentOutcome::Quota(o) => {
3946                    cumulative_wait_ms += o.duration_ms;
3947                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3948                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3949                    self.state.quota.push(loss);
3950                    self.state.event(
3951                        "fix",
3952                        format!(
3953                            "round {round}: continuation rate limited (quota); not retrying now"
3954                        ),
3955                    );
3956                    return (
3957                        seat,
3958                        None,
3959                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3960                        ContinuationRecord {
3961                            attempts,
3962                            cumulative_wait_ms,
3963                            outcome: ContinuationOutcome::QuotaLost,
3964                        },
3965                    );
3966                }
3967                AgentOutcome::Dropped(o) => {
3968                    cumulative_wait_ms += o.duration_ms;
3969                    let why = o
3970                        .dropped
3971                        .as_ref()
3972                        .map(|d| d.why.as_str())
3973                        .unwrap_or("the CLI ended the stream without delivering its answer");
3974                    last_err = format!("the CLI dropped the stream ({why})");
3975                }
3976                AgentOutcome::Failed(e) => last_err = e,
3977                // The continuation was never started; the report stays lost
3978                // and the caller records that, without any handover.
3979                AgentOutcome::ResourceBlocked(e) => {
3980                    return (
3981                        seat,
3982                        None,
3983                        Some(format!("the continuation was not started: {e}")),
3984                        ContinuationRecord {
3985                            attempts: attempts.saturating_sub(1),
3986                            cumulative_wait_ms,
3987                            outcome: ContinuationOutcome::NoSession,
3988                        },
3989                    );
3990                }
3991            }
3992        }
3993    }
3994
3995    fn after_implement(&mut self) -> Result<()> {
3996        if !self.state.candidate_preservation.is_empty() {
3997            bail!("candidate preservation incomplete; judging is unsafe");
3998        }
3999        // Scan every candidate patch once the set is complete.
4000        if self.state.leaks.is_empty() {
4001            let cfg = self.state.config.blind.clone();
4002            let mut leaks = Vec::new();
4003            for c in &self.state.candidates {
4004                let Some(patch) =
4005                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4006                else {
4007                    continue;
4008                };
4009                leaks.extend(blind::scan(
4010                    &format!("candidate {} patch", c.label),
4011                    &patch,
4012                    &cfg.vendor_tokens,
4013                ));
4014            }
4015            if !leaks.is_empty() {
4016                let summary = leaks
4017                    .iter()
4018                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4019                    .collect::<Vec<_>>()
4020                    .join(", ");
4021                match cfg.on_leak {
4022                    LeakPolicy::Fail => {
4023                        self.state.status = RunStatus::Failed;
4024                        self.state
4025                            .event("blind", format!("vendor text in a patch: {summary}"));
4026                        self.state.leaks = leaks;
4027                        self.state.save()?;
4028                        self.settle_questions();
4029                        bail!(
4030                            "blind.on_leak = \"fail\" and vendor text reached a \
4031                             judged patch: {summary}"
4032                        );
4033                    }
4034                    LeakPolicy::Redact => self.state.event(
4035                        "blind",
4036                        format!("redacting vendor text for judging: {summary}"),
4037                    ),
4038                    LeakPolicy::Warn => self.state.event(
4039                        "blind",
4040                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
4041                    ),
4042                }
4043                self.state.leaks = leaks;
4044            }
4045        }
4046
4047        if self.state.viable().is_empty() {
4048            if self.state.all_candidates_verified_noop() {
4049                // Every candidate agreed, with evidence the adoption guard
4050                // accepted, that nothing belongs in this worktree. That is
4051                // not the same fact as a candidate that simply failed to
4052                // write anything, and settling it as an ordinary `Failed`
4053                // (see `SCHEMA`'s doc for schema 10) is what let two of
4054                // task 391f's attempts burn a retry each re-discovering the
4055                // same already-landed fix. Terminal either way, so `judge`
4056                // must never run over an empty candidate set — unlike the
4057                // `Failed` branch below this returns `Ok`, not an error:
4058                // nothing here failed.
4059                self.state.status = RunStatus::VerifiedNoop;
4060                self.state.save()?;
4061                self.settle_questions();
4062                return Ok(());
4063            }
4064            self.state.status = RunStatus::Failed;
4065            self.state.save()?;
4066            self.settle_questions();
4067            bail!("no candidate produced a change; nothing to judge");
4068        }
4069        self.state.status = RunStatus::Judging;
4070        self.state.save()?;
4071        Ok(())
4072    }
4073
4074    // --------------------------------------------------------------- judge
4075
4076    async fn judge(&mut self) -> Result<()> {
4077        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4078        // agent files with `magi task add` name the run that paid for it. The
4079        // prompt overlay is cloned alongside it because the waves borrow it
4080        // while `self` is mutably borrowed by the node's own bookkeeping.
4081        let run_id = self.state.id.clone();
4082        let prompts = self.state.config.prompts.clone();
4083        if !self.state.judgements.is_empty() || self.state.judge_skipped {
4084            return Ok(());
4085        }
4086        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4087        if viable.len() == 1 {
4088            // Recorded so this is a one-time event: `judgements` stays empty
4089            // either way, which without this flag is indistinguishable from
4090            // "not yet judged" on the next reentry — and status is left
4091            // untouched, so a later node's conclusion (e.g. `Blocked` after
4092            // the review budget ran out) survives a resume instead of being
4093            // clobbered back to `Judging` by this node running again.
4094            self.state.judge_skipped = true;
4095            self.state.event(
4096                "judge",
4097                format!(
4098                    "only candidate {} produced a change; judging skipped",
4099                    viable[0].label
4100                ),
4101            );
4102            self.state.save()?;
4103            return Ok(());
4104        }
4105        self.state.status = RunStatus::Judging;
4106
4107        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4108        let language = self.state.config.graph.language.clone();
4109        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4110        let sessions = self.state.config.graph.sessions;
4111        let artifacts = agent::artifacts_dir(&self.state.dir());
4112        let root = self.state.worktree_root();
4113        let base_short = short(&self.state.base_commit);
4114
4115        let mut jobs = Vec::new();
4116        let mut orders = Vec::new();
4117        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4118            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4119            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4120            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4121            let seat_key = format!("judge-{}", j + 1);
4122            let seat = self.seat(&seat_key, &spec.id);
4123            jobs.push(SeatJob {
4124                prompt: prompt::judge(
4125                    &self.state.instruction,
4126                    &views,
4127                    self.roles.judges.len(),
4128                    &base_short,
4129                    &language,
4130                ),
4131                spec,
4132                seat,
4133                cwd: root.join(format!("judge-{}", j + 1)),
4134                timeout,
4135                allow_write: false,
4136                sessions,
4137                artifacts: artifacts.clone(),
4138                stem: format!("judge-{}", j + 1),
4139                handover: None,
4140            });
4141        }
4142
4143        self.state.event(
4144            "judge",
4145            format!(
4146                "{} judges ranking {} candidates blind",
4147                jobs.len(),
4148                viable.len()
4149            ),
4150        );
4151        let labels_for_check = labels.clone();
4152        let mut quota_losses = Vec::new();
4153        let cache = self.state.config.cache_dir();
4154        let ctx = WaveCtx {
4155            carry_seats: false,
4156            pauses: Some((&self.pause, &self.interrupt)),
4157            run: &run_id,
4158            node: "judge",
4159            prompts: &prompts,
4160            cache: cache.as_deref(),
4161            round: None,
4162        };
4163        let results = ask_json_wave::<Ranking>(
4164            jobs,
4165            Arc::clone(&self.sem),
4166            self.state.config.graph.retries,
4167            &self.roles.judge_roster,
4168            &ctx,
4169            &mut quota_losses,
4170            &mut self.state,
4171            &move |r: &Ranking| r.validate(&labels_for_check),
4172        )
4173        .await;
4174        if self.state.resource_block.is_some() {
4175            return self.stop_for_resource();
4176        }
4177        self.state.quota.extend(quota_losses);
4178
4179        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4180            let agent_id = seat.agent.clone();
4181            self.state.seats.insert(seat.key.clone(), seat);
4182            let mut record = Judgement {
4183                judge: j + 1,
4184                seat: format!("judge-{}", j + 1),
4185                agent: agent_id,
4186                ranking: Vec::new(),
4187                reasons: BTreeMap::new(),
4188                confidence: None,
4189                strengths: BTreeMap::new(),
4190                order: orders[j].clone(),
4191                failed: None,
4192                duration_ms: 0,
4193            };
4194            match res {
4195                Ok((ranking, out)) => {
4196                    record.ranking = ranking.normalized();
4197                    record.reasons = ranking.reasons;
4198                    record.strengths = ranking.strengths;
4199                    record.confidence = ranking.confidence;
4200                    record.duration_ms = out.duration_ms;
4201                    self.state.event(
4202                        "judge",
4203                        format!(
4204                            "judge {} ranked {}",
4205                            j + 1,
4206                            record.ranking.iter().collect::<String>()
4207                        ),
4208                    );
4209                }
4210                Err(e) => {
4211                    record.failed = Some(e.to_string());
4212                    self.state
4213                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4214                }
4215            }
4216            self.state.judgements.push(record);
4217            self.state.save()?;
4218        }
4219        Ok(())
4220    }
4221
4222    // ---------------------------------------------------------- deliberate
4223
4224    async fn deliberate(&mut self) -> Result<()> {
4225        if self.hold_base_sync().await? {
4226            return Ok(());
4227        }
4228        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4229        // agent files with `magi task add` name the run that paid for it. The
4230        // prompt overlay is cloned alongside it because the waves borrow it
4231        // while `self` is mutably borrowed by the node's own bookkeeping.
4232        let run_id = self.state.id.clone();
4233        let prompts = self.state.config.prompts.clone();
4234        let rounds = self.state.config.graph.deliberate_rounds;
4235        // Resume unfinished rounds after an outage, but never reopen a tally.
4236        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4237            return Ok(());
4238        }
4239        let tops: Vec<char> = self
4240            .state
4241            .judgements
4242            .iter()
4243            .filter_map(|j| j.ranking.first().copied())
4244            .collect();
4245        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4246            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4247                self.state.event(
4248                    "deliberate",
4249                    format!("judges agreed on {} outright; no deliberation", tops[0]),
4250                );
4251            }
4252            self.state.status = RunStatus::Voting;
4253            self.state.save()?;
4254            return Ok(());
4255        }
4256
4257        self.state.status = RunStatus::Deliberating;
4258        self.state.event(
4259            "deliberate",
4260            format!(
4261                "split: first choices were {} — opening {rounds} round(s)",
4262                tops.iter().collect::<String>()
4263            ),
4264        );
4265
4266        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4267        let language = self.state.config.graph.language.clone();
4268        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4269        let sessions = self.state.config.graph.sessions;
4270        let artifacts = agent::artifacts_dir(&self.state.dir());
4271        let root = self.state.worktree_root();
4272        let base_short = short(&self.state.base_commit);
4273
4274        // Judges argue in sequence so that a turn can answer the one before it;
4275        // that is the difference between deliberation and three parallel
4276        // monologues.
4277        for round in self.state.deliberation.len() + 1..=rounds {
4278            let mut turns: Vec<DeliberationTurn> = Vec::new();
4279            let mut round_losses = Vec::new();
4280            let mut asked = 0;
4281            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4282                if self.state.judgements[j].failed.is_some() {
4283                    continue;
4284                }
4285                let seat_key = format!("judge-{}", j + 1);
4286                let spec = self.occupant(&seat_key, spec);
4287                let mut seat = self.seat(&seat_key, &spec.id);
4288                let transcript = self.transcript(&turns, j);
4289                let build = |context: Option<&str>| {
4290                    prompt::deliberate(
4291                        &self.state.instruction,
4292                        context,
4293                        &transcript,
4294                        round,
4295                        rounds,
4296                        &language,
4297                    )
4298                };
4299                let block = self.candidate_block(&viable, &base_short);
4300                let full = build(Some(&block));
4301                let text = if has_context(&spec, &seat, sessions) {
4302                    build(None)
4303                } else {
4304                    full.clone()
4305                };
4306                let job = SeatJob {
4307                    spec,
4308                    seat: seat.clone(),
4309                    prompt: text,
4310                    cwd: root.join(format!("judge-{}", j + 1)),
4311                    timeout,
4312                    allow_write: false,
4313                    sessions,
4314                    artifacts: artifacts.clone(),
4315                    stem: format!("delib-{round}-judge-{}", j + 1),
4316                    handover: Some(full),
4317                };
4318                let cache = self.state.config.cache_dir();
4319                let ctx = WaveCtx {
4320                    carry_seats: false,
4321                    pauses: Some((&self.pause, &self.interrupt)),
4322                    run: &run_id,
4323                    node: "deliberate",
4324                    prompts: &prompts,
4325                    cache: cache.as_deref(),
4326                    round: None,
4327                };
4328                // A turn is never nudged (`retries` 0): a failed seat is
4329                // handed to the next roster agent, which gets the full
4330                // context. An empty answer is a turn, not a failure.
4331                let mut losses = Vec::new();
4332                asked += 1;
4333                let mut results = ask_wave_with::<String>(
4334                    vec![job],
4335                    Arc::clone(&self.sem),
4336                    0,
4337                    &self.roles.judge_roster,
4338                    &ctx,
4339                    &mut losses,
4340                    &mut self.state,
4341                    &|text: &str| {
4342                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4343                    },
4344                )
4345                .await;
4346                if self.state.resource_block.is_some() {
4347                    return self.stop_for_resource();
4348                }
4349                round_losses.extend(losses.iter().cloned());
4350                self.state.quota.extend(losses);
4351                let (updated, res, _) = results.pop().expect("one job in, one result out");
4352                seat = updated;
4353                let agent_id = seat.agent.clone();
4354                self.state.seats.insert(seat.key.clone(), seat);
4355                let body = match res {
4356                    Ok((body, _)) => body,
4357                    // Skip the seat; a CLI's raw error JSON is never read as
4358                    // this judge's position.
4359                    Err(e) => {
4360                        self.state
4361                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4362                        continue;
4363                    }
4364                };
4365                let tentative = verdict::extract_json::<Position>(&body)
4366                    .ok()
4367                    .and_then(|p| p.tentative)
4368                    .and_then(|s| s.trim().chars().next())
4369                    .map(|c| c.to_ascii_uppercase());
4370                self.state.event(
4371                    "deliberate",
4372                    format!(
4373                        "round {round}: judge {} now favours {}",
4374                        j + 1,
4375                        tentative.map_or("—".to_owned(), |c| c.to_string())
4376                    ),
4377                );
4378                turns.push(DeliberationTurn {
4379                    judge: j + 1,
4380                    agent: agent_id,
4381                    body: blind::sanitize_prose(&body, &self.state.config.blind),
4382                    tentative,
4383                });
4384            }
4385            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4386                block_machine_outage(&mut self.state, "deliberate", &round_losses);
4387                return self.stop_for_resource();
4388            }
4389            self.state
4390                .deliberation
4391                .push(DeliberationRound { round, turns });
4392            self.state.save()?;
4393        }
4394
4395        self.state.status = RunStatus::Voting;
4396        self.state.save()?;
4397        Ok(())
4398    }
4399
4400    // ---------------------------------------------------------------- vote
4401
4402    async fn vote(&mut self) -> Result<()> {
4403        if self.hold_base_sync().await? {
4404            return Ok(());
4405        }
4406        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4407        // agent files with `magi task add` name the run that paid for it. The
4408        // prompt overlay is cloned alongside it because the waves borrow it
4409        // while `self` is mutably borrowed by the node's own bookkeeping.
4410        let run_id = self.state.id.clone();
4411        let prompts = self.state.config.prompts.clone();
4412        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4413            return Ok(());
4414        }
4415        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4416        if viable.len() == 1 {
4417            return Ok(());
4418        }
4419        self.state.status = RunStatus::Voting;
4420
4421        let language = self.state.config.graph.language.clone();
4422        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4423        let sessions = self.state.config.graph.sessions;
4424        let artifacts = agent::artifacts_dir(&self.state.dir());
4425        let root = self.state.worktree_root();
4426        let base_short = short(&self.state.base_commit);
4427        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4428
4429        let mut jobs = Vec::new();
4430        let mut seats_at = Vec::new();
4431        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4432            if self
4433                .state
4434                .judgements
4435                .get(j)
4436                .is_some_and(|r| r.failed.is_some())
4437            {
4438                continue;
4439            }
4440            let seat_key = format!("judge-{}", j + 1);
4441            let spec = self.occupant(&seat_key, spec);
4442            let seat = self.seat(&seat_key, &spec.id);
4443            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4444            let text = if has_context(&spec, &seat, sessions) {
4445                prompt::final_vote(&viable, &language)
4446            } else {
4447                full.clone()
4448            };
4449            jobs.push(SeatJob {
4450                spec,
4451                seat,
4452                prompt: text,
4453                cwd: root.join(format!("judge-{}", j + 1)),
4454                timeout,
4455                allow_write: false,
4456                sessions,
4457                artifacts: artifacts.clone(),
4458                stem: format!("vote-judge-{}", j + 1),
4459                handover: Some(full),
4460            });
4461            seats_at.push(j);
4462        }
4463
4464        self.state.event(
4465            "vote",
4466            format!(
4467                "collecting {} final votes one by one, privately",
4468                jobs.len()
4469            ),
4470        );
4471        let allowed = viable.clone();
4472        let mut quota_losses = Vec::new();
4473        let cache = self.state.config.cache_dir();
4474        let ctx = WaveCtx {
4475            carry_seats: false,
4476            pauses: Some((&self.pause, &self.interrupt)),
4477            run: &run_id,
4478            node: "vote",
4479            prompts: &prompts,
4480            cache: cache.as_deref(),
4481            round: None,
4482        };
4483        let results = ask_json_wave::<FinalVote>(
4484            jobs,
4485            Arc::clone(&self.sem),
4486            self.state.config.graph.retries,
4487            &self.roles.judge_roster,
4488            &ctx,
4489            &mut quota_losses,
4490            &mut self.state,
4491            &move |v: &FinalVote| match v.label() {
4492                Some(c) if allowed.contains(&c) => Ok(()),
4493                other => bail!("vote {other:?} is not one of {allowed:?}"),
4494            },
4495        )
4496        .await;
4497        if self.state.resource_block.is_some() {
4498            return self.stop_for_resource();
4499        }
4500        self.state.quota.extend(quota_losses);
4501
4502        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4503            let agent_id = seat.agent.clone();
4504            self.state.seats.insert(seat.key.clone(), seat);
4505            let initial = self
4506                .state
4507                .judgements
4508                .get(j)
4509                .and_then(|r| r.ranking.first().copied());
4510            let mut record = VoteRecord {
4511                judge: j + 1,
4512                agent: agent_id,
4513                vote: None,
4514                reason: String::new(),
4515                changed: false,
4516            };
4517            match res {
4518                Ok((v, _)) => {
4519                    record.vote = v.label();
4520                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4521                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4522                    self.state.event(
4523                        "vote",
4524                        format!(
4525                            "judge {} voted {}{}",
4526                            j + 1,
4527                            record.vote.unwrap_or('?'),
4528                            if record.changed { " (changed)" } else { "" }
4529                        ),
4530                    );
4531                }
4532                Err(e) => {
4533                    self.state
4534                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4535                }
4536            }
4537            self.state.votes.push(record);
4538            self.state.save()?;
4539        }
4540        Ok(())
4541    }
4542
4543    // --------------------------------------------------------------- tally
4544
4545    fn tally(&mut self) -> Result<()> {
4546        if self.state.tally.is_some() {
4547            return Ok(());
4548        }
4549        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4550        let tops: Vec<char> = self
4551            .state
4552            .judgements
4553            .iter()
4554            .filter_map(|j| j.ranking.first().copied())
4555            .collect();
4556        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4557
4558        // A judge whose private vote failed still counted once, in the initial
4559        // ranking; using it beats discarding a whole seat.
4560        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4561        let mut cast: Vec<char> = Vec::new();
4562        for (i, j) in self.state.judgements.iter().enumerate() {
4563            let vote = self
4564                .state
4565                .votes
4566                .iter()
4567                .find(|v| v.judge == i + 1)
4568                .and_then(|v| v.vote)
4569                .or_else(|| j.ranking.first().copied());
4570            if let Some(v) = vote {
4571                *first_choice.entry(v).or_insert(0) += 1;
4572                cast.push(v);
4573            }
4574        }
4575
4576        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4577        for j in &self.state.judgements {
4578            let n = j.ranking.len();
4579            for (pos, label) in j.ranking.iter().enumerate() {
4580                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4581            }
4582        }
4583
4584        let best = first_choice.values().copied().max().unwrap_or(0);
4585        let mut leaders: Vec<char> = first_choice
4586            .iter()
4587            .filter(|(_, v)| **v == best)
4588            .map(|(k, _)| *k)
4589            .collect();
4590        let mut tie_break = None;
4591        if leaders.len() > 1 {
4592            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4593            let borda_leaders: Vec<char> = leaders
4594                .iter()
4595                .copied()
4596                .filter(|l| borda[l] == top_borda)
4597                .collect();
4598            tie_break = Some(if borda_leaders.len() == 1 {
4599                format!(
4600                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4601                    leaders.len()
4602                )
4603            } else {
4604                format!(
4605                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4606                    leaders.len()
4607                )
4608            });
4609            leaders = borda_leaders;
4610            leaders.sort_unstable();
4611        }
4612        let winner = *leaders
4613            .first()
4614            .or(viable.first())
4615            .context("no candidate to declare a winner from")?;
4616
4617        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4618        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4619        let deliberated = !self.state.deliberation.is_empty();
4620
4621        // Whose verdict is this? A rate-limited seat is absent even if it
4622        // ranked before the limit hit, so presence is measured against the
4623        // recorded losses, not just "did a ranking ever appear".
4624        let quota_seats: std::collections::BTreeSet<&str> =
4625            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4626        let mut present = 0usize;
4627        for (i, j) in self.state.judgements.iter().enumerate() {
4628            if quota_seats.contains(j.seat.as_str()) {
4629                continue;
4630            }
4631            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4632            let voted = self
4633                .state
4634                .votes
4635                .iter()
4636                .any(|v| v.judge == i + 1 && v.vote.is_some());
4637            if ranked || voted {
4638                present += 1;
4639            }
4640        }
4641        // Strict majority of the configured panel. A bare majority is real
4642        // signal we can act on, while a minority verdict must never stand in
4643        // for a healthy one. A one-candidate run needs no panel at all, and
4644        // `judges` stays `0` rather than the roster size a panel that never
4645        // sat would otherwise be credited with.
4646        let needs_quorum = viable.len() > 1;
4647        let judges_total = if needs_quorum {
4648            self.roles.judges.len()
4649        } else {
4650            0
4651        };
4652        let quorum = if needs_quorum {
4653            judges_total / 2 + 1
4654        } else {
4655            0
4656        };
4657        let met_quorum = !needs_quorum || present >= quorum;
4658        let uncontested = (!needs_quorum).then(|| {
4659            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4660        });
4661
4662        self.state.event(
4663            "tally",
4664            match &uncontested {
4665                Some(reason) => format!("winner {winner} — {reason}"),
4666                None => format!(
4667                    "winner {winner} — votes {} | initial {} | {} changed | \
4668                     {present}/{judges_total} judges{}",
4669                    first_choice
4670                        .iter()
4671                        .map(|(k, v)| format!("{k}:{v}"))
4672                        .collect::<Vec<_>>()
4673                        .join(" "),
4674                    if unanimous_initial {
4675                        "unanimous"
4676                    } else {
4677                        "split"
4678                    },
4679                    changed_votes,
4680                    if met_quorum {
4681                        String::new()
4682                    } else {
4683                        format!(" — below quorum ({quorum} required)")
4684                    },
4685                ),
4686            },
4687        );
4688        if !met_quorum {
4689            self.state.event(
4690                "stall",
4691                format!(
4692                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4693                     the run stops here, resumable"
4694                ),
4695            );
4696        }
4697        self.state.tally = Some(Tally {
4698            first_choice,
4699            borda,
4700            winner,
4701            rankings: tops.len(),
4702            unanimous_initial,
4703            deliberated,
4704            changed_votes,
4705            unanimous_final,
4706            tie_break,
4707            judges: judges_total,
4708            present,
4709            quorum,
4710            met_quorum,
4711            uncontested,
4712        });
4713        self.state.status = if met_quorum {
4714            RunStatus::Reviewing
4715        } else {
4716            RunStatus::Stalled
4717        };
4718        self.state.save()?;
4719        Ok(())
4720    }
4721
4722    // ------------------------------------------------------------- recover
4723
4724    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4725    /// resumed toward completion once the transient cause clears.
4726    ///
4727    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4728    /// it toward the quorum, which is exactly the set of seats whose absence
4729    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4730    /// not depend on which node happened to hit the limit), or an ordinary
4731    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4732    /// seat is never disturbed.
4733    ///
4734    /// A seat that now answers with a usable ranking is "recovered": its
4735    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4736    /// `tally` counts it present again), and its vote re-collected. A seat that
4737    /// still fails keeps its loss and stays absent.
4738    ///
4739    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4740    /// to review/gate/merge), `false` when it is still below quorum (the run
4741    /// stays `Stalled`, still resumable for a later retry).
4742    #[allow(clippy::too_many_lines)]
4743    async fn recover_stall(&mut self) -> Result<bool> {
4744        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4745        // agent files with `magi task add` name the run that paid for it. The
4746        // prompt overlay is cloned alongside it because the waves borrow it
4747        // while `self` is mutably borrowed by the node's own bookkeeping.
4748        let run_id = self.state.id.clone();
4749        let prompts = self.state.config.prompts.clone();
4750        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4751        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4752        // is what keeps a non-quota collapse — or a quota loss recorded at the
4753        // deliberate node — from being a permanent dead-end on `--resume`.
4754        let quota_seats: BTreeSet<&str> =
4755            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4756        let absent: Vec<String> = self
4757            .state
4758            .judgements
4759            .iter()
4760            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4761            .map(|j| j.seat.clone())
4762            .collect();
4763        if absent.is_empty() {
4764            return Ok(false);
4765        }
4766        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4767        if viable.len() <= 1 {
4768            return Ok(false);
4769        }
4770        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4771        let language = self.state.config.graph.language.clone();
4772        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4773        let sessions = self.state.config.graph.sessions;
4774        let artifacts = agent::artifacts_dir(&self.state.dir());
4775        let root = self.state.worktree_root();
4776        let base_short = short(&self.state.base_commit);
4777        let candidates: Vec<Candidate> = viable.clone();
4778
4779        // Map each absent seat key to its 0-based position in `roles.judges`.
4780        let mut positions: Vec<usize> = absent
4781            .iter()
4782            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4783            .collect();
4784        if positions.is_empty() {
4785            return Ok(false);
4786        }
4787        positions.sort_unstable();
4788        positions.dedup();
4789
4790        // Re-rank the lost seats, one blind prompt each.
4791        let mut judge_jobs = Vec::new();
4792        for &j in &positions {
4793            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4794            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4795            let seat_key = format!("judge-{}", j + 1);
4796            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4797            let seat = self.seat(&seat_key, &spec.id);
4798            judge_jobs.push(SeatJob {
4799                spec,
4800                seat,
4801                prompt: prompt::judge(
4802                    &self.state.instruction,
4803                    &views,
4804                    self.roles.judges.len(),
4805                    &base_short,
4806                    &language,
4807                ),
4808                cwd: root.join(seat_key),
4809                timeout,
4810                allow_write: false,
4811                sessions,
4812                artifacts: artifacts.clone(),
4813                stem: format!("judge-{}-recover", j + 1),
4814                handover: None,
4815            });
4816        }
4817
4818        let labels_for_check = labels.clone();
4819        let mut judge_losses = Vec::new();
4820        let retries = self.state.config.graph.retries;
4821        let cache = self.state.config.cache_dir();
4822        let ctx = WaveCtx {
4823            carry_seats: false,
4824            pauses: Some((&self.pause, &self.interrupt)),
4825            run: &run_id,
4826            node: "judge",
4827            prompts: &prompts,
4828            cache: cache.as_deref(),
4829            round: None,
4830        };
4831        let results = ask_json_wave::<Ranking>(
4832            judge_jobs,
4833            Arc::clone(&self.sem),
4834            retries,
4835            &self.roles.judge_roster,
4836            &ctx,
4837            &mut judge_losses,
4838            &mut self.state,
4839            &move |r: &Ranking| r.validate(&labels_for_check),
4840        )
4841        .await;
4842        if self.state.resource_block.is_some() {
4843            self.stop_for_resource()?;
4844            return Ok(false);
4845        }
4846
4847        // Refresh the judgement of every seat that ranked again.
4848        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4849        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4850            let agent_id = seat.agent.clone();
4851            self.state.seats.insert(seat.key.clone(), seat);
4852            let record = &mut self.state.judgements[j];
4853            match res {
4854                Ok((ranking, out)) => {
4855                    record.agent = agent_id;
4856                    record.ranking = ranking.normalized();
4857                    record.reasons = ranking.reasons;
4858                    record.strengths = ranking.strengths;
4859                    record.confidence = ranking.confidence;
4860                    record.failed = None;
4861                    record.duration_ms = out.duration_ms;
4862                    recovered.insert(j);
4863                    self.state.event(
4864                        "recover",
4865                        format!("judge {} ranked again after the limit", j + 1),
4866                    );
4867                }
4868                Err(e) => {
4869                    self.state
4870                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4871                }
4872            }
4873        }
4874
4875        // Re-ask the votes of the seats that recovered a ranking.
4876        let mut vote_jobs = Vec::new();
4877        let mut vote_pos: Vec<usize> = Vec::new();
4878        for &j in &recovered {
4879            let seat_key = format!("judge-{}", j + 1);
4880            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4881            let seat = self.seat(&seat_key, &spec.id);
4882            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4883            let text = if has_context(&spec, &seat, sessions) {
4884                prompt::final_vote(&labels, &language)
4885            } else {
4886                full.clone()
4887            };
4888            vote_jobs.push(SeatJob {
4889                spec,
4890                seat,
4891                prompt: text,
4892                cwd: root.join(seat_key),
4893                timeout,
4894                allow_write: false,
4895                sessions,
4896                artifacts: artifacts.clone(),
4897                stem: format!("vote-judge-{}-recover", j + 1),
4898                handover: Some(full),
4899            });
4900            vote_pos.push(j);
4901        }
4902        let allowed = labels.clone();
4903        let mut vote_losses = Vec::new();
4904        let vote_retries = self.state.config.graph.retries;
4905        let vote_cache = self.state.config.cache_dir();
4906        let ctx = WaveCtx {
4907            carry_seats: false,
4908            pauses: Some((&self.pause, &self.interrupt)),
4909            run: &run_id,
4910            node: "vote",
4911            prompts: &prompts,
4912            cache: vote_cache.as_deref(),
4913            round: None,
4914        };
4915        let votes = ask_json_wave::<FinalVote>(
4916            vote_jobs,
4917            Arc::clone(&self.sem),
4918            vote_retries,
4919            &self.roles.judge_roster,
4920            &ctx,
4921            &mut vote_losses,
4922            &mut self.state,
4923            &move |v: &FinalVote| match v.label() {
4924                Some(c) if allowed.contains(&c) => Ok(()),
4925                other => bail!("vote {other:?} is not one of {allowed:?}"),
4926            },
4927        )
4928        .await;
4929        if self.state.resource_block.is_some() {
4930            self.stop_for_resource()?;
4931            return Ok(false);
4932        }
4933        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4934            let agent_id = seat.agent.clone();
4935            self.state.seats.insert(seat.key.clone(), seat);
4936            match res {
4937                Ok((v, _)) => {
4938                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4939                        rec.vote = v.label();
4940                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4941                    } else {
4942                        self.state.votes.push(VoteRecord {
4943                            judge: j + 1,
4944                            agent: agent_id,
4945                            vote: v.label(),
4946                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4947                            changed: false,
4948                        });
4949                    }
4950                    self.state.event(
4951                        "recover",
4952                        format!("judge {} voted again after the limit", j + 1),
4953                    );
4954                }
4955                Err(e) => {
4956                    self.state
4957                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4958                }
4959            }
4960        }
4961
4962        // A seat that ranked again is present even if its re-vote failed —
4963        // `tally` falls back to the initial ranking's first choice — so clear
4964        // its quota loss. Seats that still fail keep theirs and stay absent.
4965        let recovered_keys: BTreeSet<String> = recovered
4966            .iter()
4967            .map(|&j| format!("judge-{}", j + 1))
4968            .collect();
4969        self.state
4970            .quota
4971            .retain(|q| !recovered_keys.contains(&q.seat));
4972        // A seat that hit the limit again is a fresh loss, not the old one:
4973        // replace the stale entry so the history stays one-per-seat and the
4974        // daemon can tell this attempt's loss from a previous session's.
4975        for loss in judge_losses.into_iter().chain(vote_losses) {
4976            if recovered_keys.contains(&loss.seat) {
4977                continue;
4978            }
4979            self.state.quota.retain(|q| q.seat != loss.seat);
4980            self.state.quota.push(loss);
4981        }
4982
4983        // Recompute the verdict from the refreshed panel.
4984        self.state.tally = None;
4985        self.tally()?;
4986        Ok(self
4987            .state
4988            .tally
4989            .as_ref()
4990            .map(|t| t.met_quorum)
4991            .unwrap_or(false))
4992    }
4993
4994    // ---------------------------------------------------------------- graft
4995
4996    /// Port the strengths the judges attributed to the losing candidates onto
4997    /// the winner, once, between the tally and the review loop.
4998    ///
4999    /// Best-effort by construction: it returns nothing and fails nothing. A
5000    /// skip, a fixer that failed or hit quota, or a round that changed nothing
5001    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
5002    /// the run carries on as if the phase were off. See [`graft`] for the
5003    /// decisions and `AGENTS.md` for the invariants.
5004    ///
5005    /// The one exception is a rollback that could not be confirmed: the
5006    /// rejected commit may still be the winner's tip, so the run is blocked
5007    /// (resumable; the record stays `Running`, which makes a resume retry the
5008    /// restore) instead of reviewing, gating and merging it.
5009    async fn graft(&mut self) -> Result<()> {
5010        let Err(e) = self.graft_inner().await else {
5011            return Ok(());
5012        };
5013        if e.is::<crate::repo_lock::ResourceBlocked>() {
5014            return Err(e);
5015        }
5016        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5017            self.state.status = RunStatus::Blocked;
5018            self.state.event("graft", format!("blocked: {e:#}"));
5019            self.state.save()?;
5020            return Err(e);
5021        }
5022        // An error after the `Running` record was saved may have left the
5023        // fixer's work in the tree: the record must not stay unsettled
5024        // (a later resume would reset past the review fixes), so confirm the
5025        // restore now, or block.
5026        if let Some(rec) = self.state.graft.clone()
5027            && !rec.outcome.settled()
5028        {
5029            if let Some(winner) = self.state.winner().cloned() {
5030                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5031                    if re.is::<crate::repo_lock::ResourceBlocked>() {
5032                        return Err(re);
5033                    }
5034                    self.state.status = RunStatus::Blocked;
5035                    self.state.event("graft", format!("blocked: {re:#}"));
5036                    self.state.save()?;
5037                    return Err(re);
5038                }
5039            }
5040            if let Some(g) = self.state.graft.as_mut() {
5041                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5042            }
5043        }
5044        self.state.event("graft", format!("skipped: {e:#}"));
5045        if let Err(e) = self.state.save() {
5046            tracing::warn!("save after graft: {e:#}");
5047        }
5048        Ok(())
5049    }
5050
5051    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
5052    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
5053    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5054        let _coordination = self.coordinate(wt).await?;
5055        crate::recovery::restore_repair(wt, pre)
5056            .await
5057            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5058    }
5059
5060    /// A skip is a run event, recorded once however often the node is re-entered.
5061    fn graft_skipped(&mut self, why: &str) {
5062        let message = format!("skipped: {why}");
5063        if !self
5064            .state
5065            .events
5066            .iter()
5067            .any(|e| e.node == "graft" && e.message == message)
5068        {
5069            self.state.event("graft", message);
5070        }
5071    }
5072
5073    async fn graft_inner(&mut self) -> Result<()> {
5074        // A round that was started and never finished (crash, kill): the
5075        // fixer may have left anything in the tree, so put the winner back.
5076        if let Some(rec) = self.state.graft.clone()
5077            && !rec.outcome.settled()
5078        {
5079            // Reviews exist only if the graft ended without its record being
5080            // settled (a swallowed error): the tree has moved on since, and
5081            // a reset to `pre` would discard the review fixes.
5082            if self.state.reviews.is_empty()
5083                && let Some(winner) = self.state.winner().cloned()
5084            {
5085                self.graft_restore(&winner.worktree, &rec.pre).await?;
5086            }
5087            if rec.outcome == GraftOutcome::WaitingForRestore {
5088                self.state.graft = None;
5089                self.state.save()?;
5090            } else {
5091                if let Some(g) = self.state.graft.as_mut() {
5092                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5093                }
5094                self.state.event(
5095                    "graft",
5096                    "rolled back: the previous round was interrupted; the winner is untouched",
5097                );
5098                self.state.save()?;
5099                return Ok(());
5100            }
5101        }
5102        let plan = match graft::plan(&self.state) {
5103            Ok(plan) => plan,
5104            Err(skip) => {
5105                if skip != graft::Skip::AlreadyDone {
5106                    self.graft_skipped(skip.reason());
5107                    self.state.save()?;
5108                }
5109                return Ok(());
5110            }
5111        };
5112        let Some(winner) = self.state.winner().cloned() else {
5113            return Ok(());
5114        };
5115        let wt = winner.worktree.clone();
5116        if !git::is_clean(&wt).await? {
5117            self.graft_skipped("the winner's worktree has uncommitted changes");
5118            self.state.save()?;
5119            return Ok(());
5120        }
5121        let pre = git::rev_parse(&wt, "HEAD").await?;
5122        // Saved before the fixer is called: a stop from here on is repaired
5123        // by the branch above on the next entry.
5124        self.state.graft = Some(GraftRecord {
5125            pre: pre.clone(),
5126            post: None,
5127            lines: None,
5128            limit: None,
5129            agent: None,
5130            outcome: GraftOutcome::Running,
5131            requested: graft::requested(&plan),
5132            adoption: None,
5133            adoption_error: None,
5134        });
5135        self.state.event(
5136            "graft",
5137            format!(
5138                "porting strengths of candidate(s) {} onto {}",
5139                plan.strengths
5140                    .iter()
5141                    .map(|(l, _, _)| l.to_string())
5142                    .collect::<Vec<_>>()
5143                    .join(", "),
5144                plan.winner
5145            ),
5146        );
5147        self.state.save()?;
5148
5149        let fix_prompt = prompt::graft(
5150            &self.state.instruction,
5151            plan.winner,
5152            &plan.strengths,
5153            &self.state.config.graph.language,
5154        );
5155        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5156        let sessions = self.state.config.graph.sessions;
5157        let artifacts = agent::artifacts_dir(&self.state.dir());
5158        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5159        let (_, seat, out) = self
5160            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5161                SeatJob {
5162                    prompt: fix_prompt.clone(),
5163                    spec,
5164                    seat,
5165                    cwd: wt.clone(),
5166                    timeout,
5167                    allow_write: true,
5168                    sessions,
5169                    artifacts: artifacts.clone(),
5170                    stem: "graft".to_owned(),
5171                    handover: None,
5172                }
5173            })
5174            .await;
5175        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5176            // Preserve the need to restore earlier fallback edits without
5177            // confusing an unstarted seat with an interrupted invocation.
5178            self.state.graft.as_mut().expect("graft record").outcome =
5179                GraftOutcome::WaitingForRestore;
5180            return self.stop_for_resource();
5181        }
5182        let mut why: Option<String> = None;
5183        match &out {
5184            AgentOutcome::Ok(o) if agent::output_advances(o) => {
5185                why = Some("the fixer produced nothing usable".to_owned());
5186            }
5187            AgentOutcome::Ok(_) => {}
5188            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5189            AgentOutcome::Quota(o) => {
5190                self.state.quota.push(QuotaLoss::from_report(
5191                    seat.key.clone(),
5192                    "graft",
5193                    o.quota.as_ref(),
5194                ));
5195                why = Some("rate limited (quota)".to_owned());
5196            }
5197            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5198                why = Some(format!(
5199                    "the fixer failed: {}",
5200                    e.lines()
5201                        .next()
5202                        .unwrap_or("")
5203                        .chars()
5204                        .take(160)
5205                        .collect::<String>()
5206                ));
5207            }
5208        }
5209        let agent_id = seat.agent.clone();
5210        self.state.seats.insert(seat.key.clone(), seat);
5211
5212        let mut post = None;
5213        let mut lines = None;
5214        let _coordination = self.coordinate(&wt).await?;
5215        if why.is_none() {
5216            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5217            if let Ok(r) = &rescued {
5218                self.state.note_withheld("graft", &r.withheld);
5219            }
5220            let head = git::rev_parse(&wt, "HEAD").await?;
5221            // What is measured must be what gets reviewed and gated: a failed
5222            // rescue, or anything still uncommitted (withheld lockfiles
5223            // aside, which are never part of the tree), rejects the graft.
5224            let dirty = match &rescued {
5225                Ok(r) => git::status_porcelain(&wt)
5226                    .await
5227                    .map(|st| {
5228                        st.lines().any(|l| {
5229                            !r.withheld
5230                                .iter()
5231                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
5232                        })
5233                    })
5234                    .unwrap_or(true),
5235                Err(_) => true,
5236            };
5237            if dirty {
5238                why = Some("the fixer left changes that could not be committed".to_owned());
5239            } else if head == pre {
5240                why = Some("the fixer changed nothing".to_owned());
5241            } else {
5242                lines = git::changed_lines(&wt, &pre, &head).await;
5243                post = Some(head);
5244            }
5245        }
5246        let (adoption, adoption_error) = match &out {
5247            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5248            _ => (None, None),
5249        };
5250        let requested = graft::requested(&plan);
5251        let outcome = match why {
5252            None => {
5253                self.state.event(
5254                    "graft",
5255                    format!(
5256                        "applied: {}",
5257                        lines.map_or_else(
5258                            || "changed lines unavailable".to_owned(),
5259                            |n| format!("{n} changed lines")
5260                        )
5261                    ),
5262                );
5263                GraftOutcome::Applied
5264            }
5265            Some(why) => {
5266                // Not settled until the restore is confirmed: the record
5267                // stays `Running`, so a resume retries it.
5268                crate::recovery::restore_repair(&wt, &pre)
5269                    .await
5270                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5271                self.state.event(
5272                    "graft",
5273                    format!("rolled back: {why}; the winner is untouched"),
5274                );
5275                GraftOutcome::RolledBack(why)
5276            }
5277        };
5278        self.state.graft = Some(GraftRecord {
5279            pre,
5280            post,
5281            lines,
5282            limit: None,
5283            agent: Some(agent_id),
5284            outcome,
5285            requested,
5286            adoption,
5287            adoption_error,
5288        });
5289        self.state.save()?;
5290        Ok(())
5291    }
5292
5293    // ----------------------------------------------------------------- fold
5294
5295    async fn fold_losers(&mut self) -> Result<()> {
5296        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5297            return Ok(());
5298        };
5299        if !self.state.candidate_preservation.is_empty() {
5300            bail!("candidate preservation incomplete; refusing cleanup");
5301        }
5302        let repo = self.state.repo.clone();
5303        let _coordination = self.coordinate(&repo).await?;
5304        let mut folded = Vec::new();
5305        for i in 0..self.state.candidates.len() {
5306            let c = &self.state.candidates[i];
5307            if c.label == winner || c.folded {
5308                continue;
5309            }
5310            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5311            remove_preserved_worktree(&repo, &wt).await?;
5312            if git::branch_exists(&repo, &branch).await?
5313                && !git::branch_delete(&repo, &branch).await?
5314            {
5315                bail!("candidate branch removal refused");
5316            }
5317            self.state.candidates[i].folded = true;
5318            self.state.save()?;
5319            folded.push(label.to_string());
5320        }
5321        // The judges are finished; their checkouts are pure cost from here.
5322        let root = self.state.worktree_root();
5323        for j in 1..=self.roles.judges.len() {
5324            let wt = root.join(format!("judge-{j}"));
5325            if wt.exists() {
5326                remove_preserved_worktree(&repo, &wt).await?;
5327            }
5328        }
5329        // The design-deliberation stage is finished by the time a tally
5330        // exists — same reasoning as the judges above.
5331        if self.state.config.graph.advise {
5332            for k in 1..=self.state.config.graph.advisors {
5333                let wt = root.join(format!("advisor-{k}"));
5334                if wt.exists() {
5335                    remove_preserved_worktree(&repo, &wt).await?;
5336                }
5337            }
5338        }
5339        if !folded.is_empty() {
5340            self.state
5341                .event("fold", format!("folded candidates {}", folded.join(", ")));
5342            self.state.save()?;
5343        }
5344        Ok(())
5345    }
5346
5347    // ------------------------------------------------------------ base sync
5348
5349    async fn hold_base_sync(&mut self) -> Result<bool> {
5350        self.hold_base_sync_as(None).await
5351    }
5352
5353    async fn hold_base_sync_kind(
5354        &mut self,
5355        kind: crate::base_sync_recovery::StopKind,
5356    ) -> Result<()> {
5357        self.hold_base_sync_as(Some(kind)).await?;
5358        Ok(())
5359    }
5360
5361    async fn hold_base_sync_as(
5362        &mut self,
5363        kind: Option<crate::base_sync_recovery::StopKind>,
5364    ) -> Result<bool> {
5365        if !self.state.base_sync_blocked() {
5366            self.state.base_sync_recovery.stop = None;
5367            return Ok(false);
5368        }
5369        let head = match self.state.winner() {
5370            Some(w) => git::rev_parse(&w.worktree, "HEAD")
5371                .await
5372                .unwrap_or_default(),
5373            None => String::new(),
5374        };
5375        if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5376            && !authorization.consumed
5377            && authorization.stop.head != head
5378        {
5379            authorization.refuse(format!(
5380                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5381                authorization.stop.head
5382            ));
5383        }
5384        self.state.record_base_sync_stop(&head);
5385        if let Some(kind) = kind {
5386            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5387            if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5388                stop.kind = kind;
5389            }
5390        }
5391        self.state.save()?;
5392        Ok(true)
5393    }
5394
5395    /// Land the winner's tree on the current tip of `<remote>/<base>` before
5396    /// anything verifies it.
5397    ///
5398    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
5399    /// read whatever is checked out in the winner's worktree. Left alone that
5400    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
5401    /// it when the run *branched* - and a run takes long enough that the base
5402    /// has usually moved by the time it gets here. A gate that ran there
5403    /// answers "green on the commit this run started from", not "green on
5404    /// what is about to land", and the difference showed up three times in
5405    /// one day as a green run whose merge would have reverted a file another
5406    /// pull request had already landed.
5407    ///
5408    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5409    /// `land::Step::Rebase` calls, rather than a second implementation of the
5410    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5411    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5412    /// doc warns about ("two answers to a question nobody notices until a
5413    /// diff is wrong").
5414    ///
5415    /// Foreign-checkout reviews instead merge the base into the owned shadow
5416    /// worktree, preserving the ancestry required for fast-forward delivery.
5417    /// A conflicting merge is aborted and held for operator resolution.
5418    ///
5419    /// A conflict is not the end of the road: the standing rebase is handed
5420    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5421    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5422    /// land). Once it finishes, review and the gate run as usual on the
5423    /// rebased tree, which is where a breakage the new base caused is caught
5424    /// by the ordinary gate-fix round. magi resolves nothing itself.
5425    ///
5426    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5427    /// the current durable recovery interval, bounds how often the base is
5428    /// synchronized during active recovery (lifetime attempts remain evidence).
5429    /// `rebase_fixes` is how many times a *conflict* was given to a fixer. When the fixer cannot finish the
5430    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5431    /// what was tried (rounds spent, paths still conflicted) and the branch
5432    /// and worktree stay exactly as they were - untouched, for a person to
5433    /// look at - which is also what makes re-entering this function
5434    /// afterwards a no-op instead of a second attempt at the same wall. A
5435    /// push failure ends the same way.
5436    async fn sync_to_base(&mut self) -> Result<()> {
5437        if self.state.status == RunStatus::AlreadyInBase {
5438            return Ok(());
5439        }
5440        self.hold_base_sync().await?;
5441        let conflicted = self
5442            .state
5443            .base_sync
5444            .as_ref()
5445            .is_some_and(|s| s.conflict.is_some());
5446        let Some(winner) = self.state.winner().cloned() else {
5447            return Ok(());
5448        };
5449
5450        crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5451
5452        let repo = self.state.repo.clone();
5453        let remote = self.state.config.merge.remote.clone();
5454        let base_branch = self.state.base_branch.clone();
5455        let tracking = format!("{remote}/{base_branch}");
5456
5457        let fetched_base =
5458            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5459        // No network, or the remote never had this branch: the run already
5460        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5461        // otherwise), and one that got this far is not blocked by a fetch
5462        // that fails now. It just has no newer tip to compare against.
5463        let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5464            return Ok(());
5465        };
5466
5467        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5468        self.state.ensure_base_episode();
5469        let mut pending = self
5470            .state
5471            .base_sync_recovery
5472            .operations
5473            .last()
5474            .filter(|o| !o.finished)
5475            .cloned();
5476        let scratch = self.state.dir().join("base-sync");
5477        if git::rebase_in_progress(&scratch).await {
5478            // Older resource stops predate operation reservations. Import the
5479            // standing Git operation, never retarget it to today's main.
5480            if pending.is_none()
5481                && self.state.base_sync_recovery.operations.is_empty()
5482                && !conflicted
5483                && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5484                    .await
5485                    .ok()
5486                    .as_deref()
5487                    == Some(head.as_str())
5488                && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5489                && git::is_clean(&winner.worktree).await?
5490            {
5491                let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5492                if self.state.base_sync.is_none() {
5493                    self.state.base_sync = Some(BaseSync {
5494                        tip: saved_tip.clone(),
5495                        behind: 0,
5496                        attempts: 0,
5497                        conflict: None,
5498                        already_in: None,
5499                    });
5500                }
5501                self.state.reserve_base_operation(&head, &saved_tip);
5502                self.state.event(
5503                    "base-recovery",
5504                    "imported standing legacy rebase as a spent synchronization reservation",
5505                );
5506                self.state.save()?;
5507                pending = self.state.base_sync_recovery.operations.last().cloned();
5508            }
5509            let expected = pending.as_ref().map(|op| op.tip.as_str());
5510            if expected.is_none()
5511                || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5512            {
5513                let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5514                if self.state.base_sync.is_none() {
5515                    self.state.base_sync = Some(BaseSync {
5516                        tip: tip.clone(),
5517                        behind: 0,
5518                        attempts: 0,
5519                        conflict: None,
5520                        already_in: None,
5521                    });
5522                }
5523                self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5524                self.state.event("land", why);
5525                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5526                    .await?;
5527                return Ok(());
5528            }
5529        }
5530        if let Some(op) = &pending {
5531            // A replay may add the pinned base's commits, never foreign work.
5532            let mut replay_proven =
5533                head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5534            if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5535                let wait = repository_wait(&self.state, None);
5536                let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5537                match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5538                    Ok(true) => {
5539                        self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5540                        self.state.save()?;
5541                        git::worktree_remove(&repo, &scratch).await.ok();
5542                        replay_proven = true;
5543                    }
5544                    Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5545                    Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5546                }
5547            }
5548            if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5549                if self.state.foreign_checkout.is_some() {
5550                    replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5551                        .await
5552                        .ok()
5553                        .as_deref()
5554                        == Some(op.head.as_str())
5555                        && git::rev_parse(&repo, &format!("{head}^2"))
5556                            .await
5557                            .ok()
5558                            .as_deref()
5559                            == Some(op.tip.as_str());
5560                } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5561                    .await
5562                    .is_ok_and(|m| m.is_empty())
5563                    && let Ok(added) =
5564                        crate::reconcile::origin_missing(&repo, &op.head, &head).await
5565                {
5566                    replay_proven = true;
5567                    for commit in added {
5568                        if !git::is_ancestor(&repo, &commit, &op.tip).await {
5569                            replay_proven = false;
5570                            break;
5571                        }
5572                    }
5573                    replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5574                        <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5575                }
5576            }
5577            // Never retarget an operation across a crash, even if main moved.
5578            // sync_to_head resets and cleans with -x, so even ignored untracked
5579            // files must stop recovery independently of the old tracked tree.
5580            if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5581                || (head != op.head
5582                    && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5583                        .await?
5584                        .is_empty())
5585                || (!git::is_clean(&winner.worktree).await?
5586                    && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5587                        .await
5588                        .is_ok_and(|o| o.ok()))
5589                || !replay_proven
5590            {
5591                let why = format!(
5592                    "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5593                    op.id
5594                );
5595                let sync = self
5596                    .state
5597                    .base_sync
5598                    .as_mut()
5599                    .context("reserved synchronization has no lifetime accounting")?;
5600                sync.conflict = Some(why.clone());
5601                self.state.event("land", why);
5602                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5603                    .await?;
5604                return Ok(());
5605            }
5606            if head != op.head {
5607                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5608            }
5609            tip = op.tip.clone();
5610        }
5611        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5612        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5613        if pending.is_none()
5614            && !conflicted
5615            && fetched_base
5616            && self
5617                .state
5618                .base_sync_recovery
5619                .episodes
5620                .last()
5621                .is_some_and(|e| e.boundary.is_some())
5622            && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5623            && let Ok(owner) = crate::queue::Queue::open().get(&task)
5624            && owner.execution_runs().last() == Some(&self.state.id)
5625            && owner.status != crate::queue::TaskStatus::Done
5626            && !owner.fresh_start
5627            && owner.review_branch.is_none()
5628            && crate::daemon::compatible_run_instruction(&self.state, &owner)
5629            && let Some(approval) = owner.resumption.as_ref()
5630            && approval.instruction == owner.instruction
5631            && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5632            && git::is_clean(&winner.worktree).await?
5633            && self.state.renew_base_episode(&task, approval, &head, &tip)
5634        {
5635            self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5636            self.state.save()?;
5637        }
5638
5639        // Before any rebase, and before a recorded conflict is honoured: a
5640        // branch whose change reached the base under other commit ids has
5641        // nothing to rebase and nothing to conflict with, and a run that
5642        // already stopped on that phantom conflict recovers here on resume.
5643        // `behind == 0` with head == tip is a branch the base has since taken
5644        // in whole, whether or not a conflict was ever recorded: the ancestry
5645        // proof must still run (`classify` ignores a head still on the start
5646        // commit).
5647        if (behind > 0 || conflicted || head == tip)
5648            && self
5649                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5650                .await?
5651        {
5652            return Ok(());
5653        }
5654        if conflicted && self.state.base_recovery_pending() {
5655            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5656            let owner =
5657                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5658            let owner_valid = owner.as_ref().is_ok_and(|t| {
5659                t.execution_runs().last() == Some(&self.state.id)
5660                    && t.status != crate::queue::TaskStatus::Done
5661                    && !t.fresh_start
5662                    && t.review_branch.is_none()
5663            });
5664            let invalidation = if !owner_valid {
5665                Some("latest task ownership changed")
5666            } else if !owner.as_ref().is_ok_and(|t| {
5667                authorization
5668                    .generation
5669                    .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5670            }) {
5671                Some("owner approval generation changed")
5672            } else if authorization.question.as_ref().is_some_and(|question| {
5673                !crate::base_recovery_control::decision_current(
5674                    &crate::queue::Queue::at(crate::run::home().join("queue")),
5675                    &self.state,
5676                    &authorization.task,
5677                    question,
5678                )
5679            }) {
5680                Some("recovery decision ownership, routing or stop evidence changed")
5681            } else if authorization.request.is_some()
5682                && !owner
5683                    .as_ref()
5684                    .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5685            {
5686                Some("owner decision changed")
5687            } else if let Some(scope) = &authorization.scope {
5688                if scope.tip != tip {
5689                    Some("fetched base changed")
5690                } else if !self
5691                    .state
5692                    .base_sync_recovery
5693                    .episodes
5694                    .last()
5695                    .is_some_and(|e| e.id == scope.episode)
5696                {
5697                    Some("active synchronization interval changed")
5698                } else if !owner
5699                    .as_ref()
5700                    .is_ok_and(|t| t.instruction == scope.instruction)
5701                {
5702                    Some("task instruction changed")
5703                } else {
5704                    None
5705                }
5706            } else {
5707                None
5708            };
5709            if let Some(reason) = invalidation {
5710                let authorization = self
5711                    .state
5712                    .base_sync_recovery
5713                    .authorizations
5714                    .last_mut()
5715                    .unwrap();
5716                authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5717                self.state.save()?;
5718                return Ok(());
5719            }
5720            if !fetched_base
5721                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5722                || !git::is_clean(&winner.worktree).await?
5723                || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5724            {
5725                return Ok(());
5726            }
5727        }
5728        let recovery = conflicted && self.state.consume_base_recovery(&head);
5729        if conflicted {
5730            // Consumption (including a refused changed HEAD) is durable before Git.
5731            self.state.save()?;
5732            if !recovery {
5733                return Ok(());
5734            }
5735        }
5736
5737        if behind == 0 {
5738            // A fixer-finished rebase moves the branch ref before the
5739            // winner's worktree is told (`sync_to_head` below). A run that
5740            // died in between resumes here with `behind == 0` and a tree still
5741            // holding the pre-rebase files, which review and the gate would
5742            // then read. That state is exactly: HEAD moved off the tip the
5743            // rebase started from, yet the tree is still identical to that
5744            // tip. A tree with edits of its own differs from it, so nothing
5745            // is thrown away.
5746            if let Some(from) = self
5747                .state
5748                .rebase_fixes
5749                .iter()
5750                .rev()
5751                .find_map(|r| r.from.clone())
5752                && from != head
5753                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5754                    .await
5755                    .is_ok_and(|o| o.ok())
5756            {
5757                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5758            }
5759            // An earlier attempt may have rebased the branch locally and died
5760            // before pushing it (only the fresh-rebase arm below pushes).
5761            // Publish it now, so the plain push at PR time is not refused as
5762            // a non-fast-forward. A run already holding a recorded conflict
5763            // never reaches here; that case is out of scope.
5764            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5765            if let Some(why) = &conflict {
5766                self.state.status = RunStatus::Blocked;
5767                self.state.event("land", why.clone());
5768            }
5769            self.state.base_sync = Some(BaseSync {
5770                tip,
5771                behind: 0,
5772                attempts,
5773                conflict,
5774                already_in: None,
5775            });
5776            if pending.is_some() {
5777                self.state.finish_base_operation(Some(&head));
5778            }
5779            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5780                .await?;
5781            self.state.save()?;
5782            return Ok(());
5783        }
5784
5785        if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5786            let why = format!(
5787                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5788                 synchronization(s); synchronizing again would only race it",
5789                winner.branch
5790            );
5791            self.state.status = RunStatus::Blocked;
5792            self.state.base_sync = Some(BaseSync {
5793                tip,
5794                behind,
5795                attempts,
5796                conflict: Some(why.clone()),
5797                already_in: None,
5798            });
5799            self.state.event("land", why);
5800            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5801                .await?;
5802            self.state.save()?;
5803            return Ok(());
5804        }
5805
5806        if self.state.foreign_checkout.is_some() {
5807            // Delivery to the operator's checkout is fast-forward only. A
5808            // rebase here would discard its saved head from our ancestry.
5809            let wait = repository_wait(&self.state, None);
5810            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5811            if self.state.base_sync.is_none() {
5812                self.state.base_sync = Some(BaseSync {
5813                    tip: tip.clone(),
5814                    behind,
5815                    attempts: 0,
5816                    conflict: None,
5817                    already_in: None,
5818                });
5819            }
5820            self.state.reserve_base_operation(&head, &tip);
5821            self.state.save()?;
5822            let merged = git::git_raw(
5823                &winner.worktree,
5824                &[
5825                    "-c",
5826                    "core.hooksPath=/dev/null",
5827                    "merge",
5828                    "--no-ff",
5829                    "--no-autostash",
5830                    "--no-edit",
5831                    "-m",
5832                    &format!("Merge {tracking} before verifying"),
5833                    &tip,
5834                ],
5835            )
5836            .await?;
5837            let conflict = if merged.ok() {
5838                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5839                self.state.event(
5840                    "land",
5841                    format!("merged {tracking} into {} before verifying", winner.branch),
5842                );
5843                None
5844            } else {
5845                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5846                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5847                }
5848                let why = format!(
5849                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5850                    winner.branch,
5851                    merged.stderr.chars().take(600).collect::<String>()
5852                );
5853                self.state.status = RunStatus::Blocked;
5854                self.state.event("land", &why);
5855                Some(why)
5856            };
5857            let synced = conflict.is_none();
5858            let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5859            self.state
5860                .finish_base_operation(synced.then_some(fresh.as_str()));
5861            let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5862            self.state.base_sync = Some(BaseSync {
5863                tip,
5864                behind: if synced { 0 } else { behind },
5865                attempts,
5866                conflict,
5867                already_in: None,
5868            });
5869            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5870                .await?;
5871            self.state.save()?;
5872            return Ok(());
5873        }
5874
5875        self.state.event(
5876            "land",
5877            format!(
5878                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5879                winner.branch
5880            ),
5881        );
5882        self.state.save()?;
5883
5884        // The remote's copy of the branch, read now and only if the fetch
5885        // really succeeded (a stale tracking ref must never pin a lease). It is
5886        // pushed over after a rebase only when it is a commit this branch
5887        // already contains, by ancestry or by patch (an earlier rebase of ours
5888        // that never reached the remote): anything else is somebody else's work.
5889        let branch_tracking = format!("{remote}/{}", winner.branch);
5890        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5891        let remote_tip = if self.state.task_pr_branch().is_none()
5892            && matches!(&fetched_branch, Ok(o) if o.ok())
5893        {
5894            git::rev_parse(&repo, &branch_tracking).await.ok()
5895        } else {
5896            None
5897        };
5898        // A remote tip this branch does not contain is somebody else's work:
5899        // rebasing would leave a local tip that can never be pushed. Stop
5900        // before touching anything and say so.
5901        if let Some(theirs) = &remote_tip
5902            && !git::is_ancestor(&repo, theirs, &head).await
5903            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5904                .await
5905                .is_ok_and(|missing| missing.is_empty())
5906        {
5907            let why = format!(
5908                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5909                 them",
5910                short(theirs),
5911                winner.branch
5912            );
5913            self.state.status = RunStatus::Blocked;
5914            self.state.base_sync = Some(BaseSync {
5915                tip,
5916                behind,
5917                attempts,
5918                conflict: Some(why.clone()),
5919                already_in: None,
5920            });
5921            self.state.event("land", why);
5922            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5923                .await?;
5924            self.state.save()?;
5925            return Ok(());
5926        }
5927
5928        if self.state.base_sync.is_none() {
5929            self.state.base_sync = Some(BaseSync {
5930                tip: tip.clone(),
5931                behind,
5932                attempts: 0,
5933                conflict: None,
5934                already_in: None,
5935            });
5936        }
5937        self.state.reserve_base_operation(&head, &tip);
5938        self.state.save()?;
5939        let scratch = self.state.dir().join("base-sync");
5940        let rebased =
5941            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5942                .await
5943            {
5944                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5945                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5946                Err(e) => Err(e),
5947            };
5948        let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5949        let stop_kind = match &rebased {
5950            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5951            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5952            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5953        };
5954        match rebased {
5955            Ok(None) => {
5956                // The branch ref moved, but a worktree that already had it
5957                // checked out (the winner's) was not told; sync its index and
5958                // files before anything reads them.
5959                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5960                let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5961                self.state.finish_base_operation(Some(&fresh));
5962                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5963                let mut conflict = None;
5964                if let Some(pinned) = &remote_tip {
5965                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5966                    match pushed {
5967                        Ok(o) if o.ok() => self.state.event(
5968                            "land",
5969                            format!("pushed rebased {} to {remote}", winner.branch),
5970                        ),
5971                        Ok(o) => {
5972                            conflict = Some(format!(
5973                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5974                                winner.branch,
5975                                short(pinned),
5976                                o.stderr.chars().take(600).collect::<String>()
5977                            ));
5978                        }
5979                        Err(e) => {
5980                            conflict = Some(format!(
5981                                "rebased {} locally but could not push it: {e:#}",
5982                                winner.branch
5983                            ));
5984                        }
5985                    }
5986                }
5987                if let Some(why) = &conflict {
5988                    self.state.status = RunStatus::Blocked;
5989                    self.state.event("land", why.clone());
5990                }
5991                self.state.base_sync = Some(BaseSync {
5992                    tip: tip.clone(),
5993                    behind: 0,
5994                    attempts,
5995                    conflict,
5996                    already_in: None,
5997                });
5998                self.state
5999                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
6000            }
6001            Ok(Some(conflict)) => {
6002                self.state.finish_base_operation(None);
6003                let why = format!(
6004                    "{} conflicts with {tracking} and did not rebase: {}",
6005                    winner.branch,
6006                    conflict.chars().take(600).collect::<String>()
6007                );
6008                self.state.status = RunStatus::Blocked;
6009                self.state.base_sync = Some(BaseSync {
6010                    tip,
6011                    behind,
6012                    attempts,
6013                    conflict: Some(why.clone()),
6014                    already_in: None,
6015                });
6016                self.state.event("land", why);
6017            }
6018            // A refused Git write or a busy repository: nothing is wrong with
6019            // the branch. The scratch rebase stays standing and the run
6020            // resumes into it (`base_sync` must not claim a conflict).
6021            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6022                self.state.save()?;
6023                return Ok(());
6024            }
6025            Err(e) => {
6026                self.state.finish_base_operation(None);
6027                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6028                self.state.status = RunStatus::Blocked;
6029                self.state.base_sync = Some(BaseSync {
6030                    tip,
6031                    behind,
6032                    attempts,
6033                    conflict: Some(why.clone()),
6034                    already_in: None,
6035                });
6036                self.state.event("land", why);
6037            }
6038        }
6039        if !self.state.base_sync_blocked() {
6040            self.state.base_sync_recovery.stop = None;
6041        }
6042        self.hold_base_sync_kind(stop_kind).await?;
6043        self.state.save()?;
6044        Ok(())
6045    }
6046
6047    /// Push a branch an earlier attempt rebased locally but never published,
6048    /// pinned to the remote tip read right after a successful fetch. Returns
6049    /// the reason when the run must stop; `None` when there was nothing to do
6050    /// (no remote copy, the same tip, or a remote copy this branch already
6051    /// contains, which the PR-time push fast-forwards) or the push succeeded.
6052    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6053        // Task competitions publish only after the PR text gate, to their
6054        // public head; never resurrect a candidate branch on the remote.
6055        if self.state.task_pr_branch().is_some() {
6056            return None;
6057        }
6058        let repo = self.state.repo.clone();
6059        let remote = self.state.config.merge.remote.clone();
6060        let fetched = git::fetch(&repo, &remote, branch).await;
6061        if !matches!(&fetched, Ok(o) if o.ok()) {
6062            return None;
6063        }
6064        let branch_tracking = format!("{remote}/{branch}");
6065        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6066        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6067            return None;
6068        }
6069        if !crate::reconcile::origin_missing(&repo, head, &theirs)
6070            .await
6071            .is_ok_and(|missing| missing.is_empty())
6072        {
6073            return Some(format!(
6074                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6075                 them",
6076                short(&theirs)
6077            ));
6078        }
6079        match git::push_pinned(&repo, &remote, branch, &theirs).await {
6080            Ok(o) if o.ok() => {
6081                self.state
6082                    .event("land", format!("pushed rebased {branch} to {remote}"));
6083                None
6084            }
6085            Ok(o) => Some(format!(
6086                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6087                 someone may have pushed): {}",
6088                short(&theirs),
6089                o.stderr.chars().take(600).collect::<String>()
6090            )),
6091            Err(e) => Some(format!(
6092                "{branch} is rebased locally but could not be pushed: {e:#}"
6093            )),
6094        }
6095    }
6096
6097    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
6098    /// is already on `tip` under other commit ids ([`crate::already`]); returns
6099    /// whether it did.
6100    ///
6101    /// Checked only when the base is ahead of the branch. A failing check is
6102    /// "not proven" - the ordinary rebase path then decides - never a reason to
6103    /// stop the run.
6104    ///
6105    /// The remote copy of the branch is held to the same standard as the local
6106    /// one: if it carries a tip this worktree does not, that tip must itself be
6107    /// proven in the base, or nothing is settled (a pull request would
6108    /// otherwise be closed over commits nobody checked). The pull request is
6109    /// closed *before* the terminal status is saved; if that fails for a
6110    /// reason other than a refusal (no network, a `gh` error) the run is left
6111    /// `Blocked` with the reason as its conflict, which a resume retries -
6112    /// the same recovery a phantom conflict gets.
6113    async fn settle_already_in(
6114        &mut self,
6115        branch: &str,
6116        tip: &str,
6117        head: &str,
6118        attempts: usize,
6119        behind: usize,
6120    ) -> Result<bool> {
6121        let repo = self.state.repo.clone();
6122        let remote = self.state.config.merge.remote.clone();
6123        let start = self.state.base_commit.clone();
6124        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6125            Ok(Some(e)) => e,
6126            Ok(None) => return Ok(false),
6127            Err(e) => {
6128                tracing::warn!("already-in-base check for {branch}: {e:#}");
6129                return Ok(false);
6130            }
6131        };
6132        let mut verified = vec![head.to_owned()];
6133        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6134        let fetched = git::fetch(&repo, &remote, &public_branch).await;
6135        if matches!(&fetched, Ok(o) if o.ok())
6136            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6137            && theirs != head
6138        {
6139            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6140                Ok(Some(_)) => verified.push(theirs),
6141                _ => return Ok(false),
6142            }
6143        }
6144        let base_branch = self.state.base_branch.clone();
6145        let message = format!(
6146            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6147             land",
6148            evidence.names(),
6149            evidence.proof.as_str()
6150        );
6151        let closed =
6152            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6153                .await;
6154        match closed {
6155            Ok(Ok(url)) => self
6156                .state
6157                .event("land", format!("closed {url}: superseded on {base_branch}")),
6158            Ok(Err(why)) => self
6159                .state
6160                .event("land", format!("did not close a pull request: {why}")),
6161            Err(e) => {
6162                let why = format!(
6163                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
6164                     not be closed ({e:#}); resume to retry"
6165                );
6166                self.state.status = RunStatus::Blocked;
6167                self.state.base_sync = Some(BaseSync {
6168                    tip: tip.to_owned(),
6169                    behind,
6170                    attempts,
6171                    conflict: Some(why.clone()),
6172                    already_in: None,
6173                });
6174                self.state.event("land", why);
6175                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6176                    .await?;
6177                self.state.save()?;
6178                return Ok(true);
6179            }
6180        }
6181        self.state.status = RunStatus::AlreadyInBase;
6182        self.state.base_sync_recovery.stop = None;
6183        self.state.base_sync = Some(BaseSync {
6184            tip: tip.to_owned(),
6185            behind,
6186            attempts,
6187            conflict: None,
6188            already_in: Some(evidence),
6189        });
6190        self.state.event("land", message);
6191        self.state.save()?;
6192        self.settle_questions();
6193        Ok(true)
6194    }
6195
6196    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
6197    /// last landed the winner on, once it has run, else the commit the run
6198    /// branched from.
6199    ///
6200    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
6201    /// and `vote` all happen before there is a winner to rebase, so they
6202    /// compare every candidate against the branch point on purpose, and a
6203    /// base that moves after they are already done cannot change an answer
6204    /// they already gave.
6205    fn landing_base(&self) -> String {
6206        self.state
6207            .base_sync
6208            .as_ref()
6209            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6210    }
6211
6212    // ------------------------------------------------------- operator fix
6213
6214    /// Route specific, already-recorded review findings to a fixer for a
6215    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
6216    /// entry point.
6217    ///
6218    /// Distinct from `review_loop`'s own fix step in three ways: it never
6219    /// runs a reviewer wave, it never spends review-round budget, and what
6220    /// happened is recorded as an [`OperatorFixRequest`] appended to
6221    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
6222    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
6223    /// and vote must never be rewritten to look like a manufactured blocking
6224    /// verdict.
6225    ///
6226    /// Only meaningful once review has actually concluded: `Ready` (handed
6227    /// off with findings still open, or simply concluded clean while minor
6228    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
6229    /// gate failed). Everything else is refused: a run still in progress
6230    /// should simply be resumed, and a `Merged` run's branch has already
6231    /// landed — reopening *this* run's own record cannot change that, so the
6232    /// answer there is a fresh `magi review <branch>`.
6233    ///
6234    /// A real commit here re-verifies through a fresh, ordinary review-only
6235    /// run on the same branch ([`Self::review`]) rather than reopening this
6236    /// run's own `review_loop`: once any round in this run's history went
6237    /// clean, `review_conclusion` treats that as permanent by design (the
6238    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
6239    /// way to force one more genuine reviewer wave out of *this* run without
6240    /// either rewriting history or weakening that guarantee for every other
6241    /// caller. A review-only run costs nothing extra — no implementation, no
6242    /// judging, no vote — and exercises the exact same review → verify →
6243    /// gate → (human) merge path, unmodified.
6244    pub async fn fix_selected(
6245        &mut self,
6246        ids: &[String],
6247        reason: &str,
6248        allow_stale: bool,
6249    ) -> Result<()> {
6250        let reason = reason.trim();
6251        if reason.is_empty() {
6252            bail!("a fix request needs a reason — that is the operator's own record of why");
6253        }
6254        if ids.is_empty() {
6255            bail!("no finding id given");
6256        }
6257        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6258            bail!(
6259                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6260                 has already concluded — can be given a targeted fix. A run still \
6261                 in progress should simply be resumed; a `merged` run's branch has \
6262                 already landed, so its answer is a fresh `magi review <branch>`, \
6263                 not reopening this run's own record",
6264                self.state.id,
6265                self.state.status.as_str()
6266            );
6267        }
6268        // Reassignment shares this claim through its atomic task save. Check
6269        // the canonical run identity inside it, before preflight or any await.
6270        let _claim = FixClaim::acquire(&self.state.dir())?;
6271        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6272        let Some(winner) = self.state.winner().cloned() else {
6273            bail!("run {} has no winning candidate to fix", self.state.id);
6274        };
6275        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6276            bail!(
6277                "branch `{}` no longer exists; this run cannot be extended",
6278                winner.branch
6279            );
6280        }
6281        let home = crate::run::home();
6282        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6283            bail!(
6284                "run {} is currently being worked on by another magi process",
6285                self.state.id
6286            );
6287        }
6288        // Resolve every id before spending anything — an unknown id refuses
6289        // the whole request rather than silently dropping it — and dedup
6290        // while keeping the operator's own order.
6291        let mut seen = BTreeSet::new();
6292        let mut findings = Vec::new();
6293        let mut missing = Vec::new();
6294        for id in ids {
6295            if !seen.insert(id.clone()) {
6296                continue;
6297            }
6298            match self.state.finding(id) {
6299                Some((round, rec, f)) => findings.push(OperatorFixFinding {
6300                    id: f.id.clone(),
6301                    severity: f.severity,
6302                    reviewer_vote: rec.vote,
6303                    round: round.round,
6304                    round_head: round.head.clone(),
6305                    reviewer: rec.reviewer,
6306                    agent: rec.agent.clone(),
6307                    file: f.file.clone(),
6308                    line: f.line,
6309                    title: f.title.clone(),
6310                    detail: f.detail.clone(),
6311                    outcome: OperatorFixOutcome::Pending,
6312                }),
6313                None => missing.push(id.clone()),
6314            }
6315        }
6316        if !missing.is_empty() {
6317            bail!(
6318                "unknown finding id(s): {}; nothing was changed",
6319                missing.join(", ")
6320            );
6321        }
6322
6323        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6324        let stale_details: Vec<(String, String)> = findings
6325            .iter()
6326            .filter(|f| f.round_head != head_at_request)
6327            .map(|f| (f.id.clone(), f.round_head.clone()))
6328            .collect();
6329        let stale = !stale_details.is_empty();
6330        if stale && !allow_stale {
6331            bail!(
6332                "the branch has moved since some finding(s) were raised — {} — now \
6333                 at {}; pass --allow-stale to fix anyway, or re-run review first",
6334                stale_details
6335                    .iter()
6336                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6337                    .collect::<Vec<_>>()
6338                    .join(", "),
6339                short(&head_at_request)
6340            );
6341        }
6342
6343        let request = OperatorFixRequest {
6344            requested_at: Timestamp::now(),
6345            reason: reason.to_owned(),
6346            findings,
6347            head_at_request: head_at_request.clone(),
6348            allow_stale,
6349            stale,
6350            fix: None,
6351            result_head: None,
6352            follow_up_review_run: None,
6353        };
6354        self.state.event(
6355            "fix",
6356            format!(
6357                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6358                request.findings.len(),
6359                request
6360                    .findings
6361                    .iter()
6362                    .map(|f| f.id.as_str())
6363                    .collect::<Vec<_>>()
6364                    .join(", "),
6365            ),
6366        );
6367        // Recorded now, before any worktree work or the fixer call itself —
6368        // and re-saved at each checkpoint below: a crash at any point after
6369        // this (mid fixer call, mid follow-up review) must not lose the fact
6370        // that this was requested, for which findings, and why. Everything
6371        // past this point reads and writes through `request_index` rather
6372        // than a local variable, since `request` itself is moved here.
6373        self.state.operator_fixes.push(request);
6374        self.state.save()?;
6375        let request_index = self.state.operator_fixes.len() - 1;
6376
6377        // A fresh, dedicated worktree for this one call, never the winner's
6378        // own worktree in place: that one may already be gone (folded away),
6379        // and reusing it in place would leave the branch checked out there
6380        // when the follow-up review below tries to check it out again. Freed
6381        // immediately after, either way — but only once confirmed clean:
6382        // `worktree_remove` is a `git worktree remove --force`, which would
6383        // otherwise discard uncommitted work left there by the operator or
6384        // another process before this had a chance to even look at it.
6385        let repo = self.state.repo.clone();
6386        let coordination = self.coordinate(&repo).await?;
6387        if winner.worktree.exists() {
6388            // Lockfiles a rescue commit withheld stay untracked on purpose and
6389            // are already recorded; they are not the operator's work to protect.
6390            let dirty = git::git(
6391                &winner.worktree,
6392                &["status", "--porcelain", "--untracked-files=all"],
6393            )
6394            .await?;
6395            let only_withheld = dirty.lines().all(|l| {
6396                l.strip_prefix("?? ")
6397                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6398            });
6399            if !only_withheld {
6400                bail!(
6401                    "`{}` has uncommitted changes; refusing to touch it — commit or \
6402                     discard them first",
6403                    winner.worktree.display()
6404                );
6405            }
6406            git::worktree_remove(&self.state.repo, &winner.worktree)
6407                .await
6408                .ok();
6409        }
6410        let fix_worktree = self.state.worktree_root().join("operator-fix");
6411        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6412        git::git(
6413            &self.state.repo,
6414            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6415        )
6416        .await
6417        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6418        if !git::is_clean(&fix_worktree).await? {
6419            git::worktree_remove(&self.state.repo, &fix_worktree)
6420                .await
6421                .ok();
6422            bail!(
6423                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6424                winner.branch
6425            );
6426        }
6427        let repo_for_setup = self.state.repo.clone();
6428        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6429            Ok(()) => {
6430                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6431            }
6432            Err(e) => Err(e),
6433        };
6434        if let Err(e) = setup {
6435            git::worktree_remove(&self.state.repo, &fix_worktree)
6436                .await
6437                .ok();
6438            return Err(e);
6439        }
6440
6441        // This explicit operator request retries availability independently
6442        // of execute(); the previous call has already saved and cleaned up.
6443        self.state.resource_block = None;
6444
6445        let run_id = self.state.id.clone();
6446        let prompts = self.state.config.prompts.clone();
6447        let language = self.state.config.graph.language.clone();
6448        let sessions = self.state.config.graph.sessions;
6449        let artifacts = agent::artifacts_dir(&self.state.dir());
6450        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6451            .findings
6452            .iter()
6453            .map(|f| Finding {
6454                id: f.id.clone(),
6455                severity: f.severity,
6456                file: f.file.clone(),
6457                line: f.line,
6458                title: f.title.clone(),
6459                detail: f.detail.clone(),
6460            })
6461            .collect();
6462        let fix_prompt = prompt::operator_fix_in(
6463            &self.state.instruction,
6464            &finding_list,
6465            reason,
6466            &stale_details,
6467            &head_at_request,
6468            &language,
6469            &self.state.config.graph.github_language,
6470        );
6471        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6472        drop(coordination);
6473        let (job, seat, out) = self
6474            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6475                prompt: fix_prompt.clone(),
6476                spec,
6477                seat,
6478                cwd: fix_worktree.clone(),
6479                timeout,
6480                allow_write: true,
6481                sessions,
6482                artifacts: artifacts.clone(),
6483                stem: "operator-fix".to_owned(),
6484                handover: None,
6485            })
6486            .await;
6487        let agent_id = seat.agent.clone();
6488
6489        let mut fix = FixRecord {
6490            agent: agent_id,
6491            addressed: Vec::new(),
6492            rejected: Vec::new(),
6493            notes: String::new(),
6494            committed: false,
6495            failed: None,
6496            duration_ms: 0,
6497            continuation: None,
6498        };
6499        let mut final_seat = seat.clone();
6500        match out {
6501            AgentOutcome::Ok(o) => {
6502                fix.duration_ms = o.duration_ms;
6503                let parsed = verdict::extract_json::<FixReport>(&o.text);
6504                let incomplete_reason = match &parsed {
6505                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6506                        "the reply parsed, but it reported a command whose own CLI \
6507                         never confirmed an exit status"
6508                            .to_owned(),
6509                    ),
6510                    Ok(_) => None,
6511                    Err(e) => Some(e.to_string()),
6512                };
6513                match incomplete_reason {
6514                    None => {
6515                        let report = parsed.expect("checked Ok above");
6516                        fix.addressed = report.addressed;
6517                        fix.rejected = report.rejected;
6518                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6519                    }
6520                    Some(reason) => {
6521                        let (resumed_seat, resolved, failure, cont) = self
6522                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6523                            .await;
6524                        fix.duration_ms += cont.cumulative_wait_ms;
6525                        fix.continuation = Some(cont);
6526                        final_seat = resumed_seat;
6527                        match resolved {
6528                            Some(report) => {
6529                                fix.addressed = report.addressed;
6530                                fix.rejected = report.rejected;
6531                                fix.notes =
6532                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6533                            }
6534                            None => fix.failed = failure,
6535                        }
6536                    }
6537                }
6538            }
6539            AgentOutcome::Dropped(o) => {
6540                fix.duration_ms = o.duration_ms;
6541                let why = o
6542                    .dropped
6543                    .as_ref()
6544                    .map(|d| d.why.as_str())
6545                    .unwrap_or("the CLI ended the stream without delivering its answer");
6546                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6547            }
6548            AgentOutcome::Quota(o) => {
6549                self.state.quota.push(QuotaLoss::from_report(
6550                    final_seat.key.clone(),
6551                    "fix",
6552                    o.quota.as_ref(),
6553                ));
6554                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6555            }
6556            AgentOutcome::Failed(e) => fix.failed = Some(e),
6557            AgentOutcome::ResourceBlocked(e) => {
6558                fix.failed = Some(format!("the fixer was not started: {e}"));
6559            }
6560        }
6561        // Even a deferred call must rescue edits, release the worktree and
6562        // finalize its record below so an explicit `magi fix` can retry.
6563        if fix.continuation.is_none() {
6564            fix.continuation = Some(ContinuationRecord::not_needed());
6565        }
6566        self.state.seats.insert(final_seat.key.clone(), final_seat);
6567
6568        let rescue_message = format!(
6569            "magi: operator-selected fix ({}) (uncommitted work)",
6570            self.state.operator_fixes[request_index]
6571                .findings
6572                .iter()
6573                .map(|f| f.id.as_str())
6574                .collect::<Vec<_>>()
6575                .join(", ")
6576        );
6577        let repo = self.state.repo.clone();
6578        let coordination = self.coordinate(&repo).await?;
6579        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6580            self.state.note_withheld("fix", &r.withheld);
6581        }
6582        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6583        fix.committed = after != head_at_request;
6584        git::worktree_remove(&self.state.repo, &fix_worktree)
6585            .await
6586            .ok();
6587        drop(coordination);
6588
6589        self.state.event(
6590            "fix",
6591            match &fix.failed {
6592                Some(reason) => format!(
6593                    "operator fix: adoption report was lost ({reason}); {}",
6594                    if fix.committed {
6595                        "committed"
6596                    } else {
6597                        "NO new commit"
6598                    }
6599                ),
6600                None => format!(
6601                    "operator fix: {} addressed, {} rejected, {}",
6602                    fix.addressed.len(),
6603                    fix.rejected.len(),
6604                    if fix.committed {
6605                        "committed"
6606                    } else {
6607                        "NO new commit"
6608                    }
6609                ),
6610            },
6611        );
6612
6613        // Every selected finding gets an outcome — never left `Pending` once
6614        // the fixer's own turn is over. A report that never came back at all
6615        // marks every one of them `Unreported`, not silently "not addressed":
6616        // quota, a dropped stream, or an exhausted continuation are gaps in
6617        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
6618        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
6619        for f in &mut self.state.operator_fixes[request_index].findings {
6620            f.outcome = if fix.failed.is_some() {
6621                OperatorFixOutcome::Unreported
6622            } else if fix.addressed.contains(&f.id) {
6623                OperatorFixOutcome::Addressed
6624            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6625                OperatorFixOutcome::Rejected { why: r.why.clone() }
6626            } else {
6627                OperatorFixOutcome::Unreported
6628            };
6629        }
6630
6631        let committed = fix.committed;
6632        if committed {
6633            self.state.operator_fixes[request_index].result_head = Some(after.clone());
6634        }
6635        self.state.operator_fixes[request_index].fix = Some(fix);
6636        // Saved again now that the fixer's own outcome is final, on top of
6637        // the save right after the request was first pushed above.
6638        self.state.save()?;
6639        if self.state.resource_block.is_some() {
6640            return self.stop_for_resource();
6641        }
6642
6643        if committed {
6644            self.state.event(
6645                "fix",
6646                format!(
6647                    "operator fix committed {}; opening a follow-up review-only run",
6648                    short(&after)
6649                ),
6650            );
6651            // The operator asked for the fix, and the follow-up serves whatever
6652            // task the run it follows served.
6653            let origin =
6654                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6655            match Self::review(
6656                &self.state.repo,
6657                &winner.branch,
6658                self.state.config.clone(),
6659                origin,
6660            )
6661            .await
6662            {
6663                Ok(mut follow_up) => {
6664                    follow_up.state.event(
6665                        "start",
6666                        format!(
6667                            "requested by an operator fix on run {} for finding(s) {}",
6668                            self.state.id,
6669                            self.state.operator_fixes[request_index]
6670                                .findings
6671                                .iter()
6672                                .map(|f| f.id.as_str())
6673                                .collect::<Vec<_>>()
6674                                .join(", "),
6675                        ),
6676                    );
6677                    follow_up.state.save()?;
6678                    let follow_up_id = follow_up.state.id.clone();
6679                    // The follow-up is a run like any other: it belongs to the
6680                    // task of the run it follows, or to one filed for it.
6681                    let adopted = match crate::direct::adopt(
6682                        &follow_up.state,
6683                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6684                    ) {
6685                        Ok(a) => a,
6686                        Err(e) => {
6687                            // An ownerless run must not spend agent calls; it
6688                            // stays saved, and `magi run --resume` adopts it.
6689                            self.state.event(
6690                                "fix",
6691                                format!(
6692                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6693                                ),
6694                            );
6695                            self.state.save()?;
6696                            return Ok(());
6697                        }
6698                    };
6699                    let executed = follow_up.execute().await;
6700                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6701                    if let Some(a) = adopted {
6702                        a.finish(&follow_up.state, executed);
6703                    }
6704                    if let Some(e) = failure {
6705                        self.state.event(
6706                            "fix",
6707                            format!(
6708                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6709                            ),
6710                        );
6711                    }
6712                    self.state.operator_fixes[request_index].follow_up_review_run =
6713                        Some(follow_up_id);
6714                }
6715                Err(e) => {
6716                    self.state.event(
6717                        "fix",
6718                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6719                    );
6720                }
6721            }
6722            self.state.save()?;
6723        }
6724
6725        Ok(())
6726    }
6727
6728    // --------------------------------------------------------------- review
6729
6730    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6731    /// names, in order, each at most once, moving on only when the call
6732    /// advances (an error, a quota hit or nothing usable - the same decision
6733    /// point `agent::chain_advances` is for the other chained roles).
6734    ///
6735    /// Each agent gets its own seat from `Runner::seat`: the same agent
6736    /// continues its conversation (the winner's own implementer seat when it
6737    /// is the winner's author, now that the competition is over), another
6738    /// takes a fresh one so the full prompt is sent again. A handover is
6739    /// recorded under `node`, which is what makes the fallback stick for the
6740    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6741    /// is returned, so an exhausted chain reads exactly like a single failed
6742    /// fixer: one quota loss, the same wording, the same refund. An earlier
6743    /// attempt's edits are left in the tree and judged, with the final
6744    /// attempt's, by what git says afterwards.
6745    ///
6746    /// Returns the job that produced the outcome, for `continue_fix_report`.
6747    async fn ask_fixer(
6748        &mut self,
6749        winner: &Candidate,
6750        node: &'static str,
6751        round: Option<usize>,
6752        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6753    ) -> (SeatJob, SeatState, AgentOutcome) {
6754        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6755        self.ask_fixer_among(attempts, node, round, None, build)
6756            .await
6757    }
6758
6759    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6760    /// worktree and a commit: before the next agent is asked after one that
6761    /// did not satisfy the call, the worktree is put back on that commit, so
6762    /// a failed agent's partial edits are not carried into the next one.
6763    async fn ask_fixer_among(
6764        &mut self,
6765        attempts: Vec<(AgentSpec, String)>,
6766        node: &'static str,
6767        round: Option<usize>,
6768        restore: Option<(&Path, &str)>,
6769        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6770    ) -> (SeatJob, SeatState, AgentOutcome) {
6771        let run_id = self.state.id.clone();
6772        let prompts = self.state.config.prompts.clone();
6773        let cache = self.state.config.cache_dir();
6774        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6775        let mut last = None;
6776        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6777            let seat = self.seat(&key, &spec.id);
6778            let mut job = build(spec.clone(), seat);
6779            if i > 0 {
6780                job.stem = format!("{}-{}", job.stem, spec.id);
6781            }
6782            let ctx = WaveCtx {
6783                carry_seats: false,
6784                pauses: Some((&self.pause, &self.interrupt)),
6785                run: &run_id,
6786                node,
6787                prompts: &prompts,
6788                cache: cache.as_deref(),
6789                round,
6790            };
6791            let (seat, out) =
6792                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6793            // A resource block never advances the chain: the chosen agent
6794            // was not asked, so it must be the one asked again on resume.
6795            let advances = match &out {
6796                AgentOutcome::Ok(o) => agent::output_advances(o),
6797                AgentOutcome::ResourceBlocked(_) => false,
6798                _ => true,
6799            };
6800            if let Some(next) = ids.get(i + 1)
6801                && advances
6802            {
6803                self.state.seats.insert(seat.key.clone(), seat.clone());
6804                let class =
6805                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6806                let reason = match &out {
6807                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6808                    other => fail_reason(other),
6809                };
6810                // A failed agent's edits must not reach the next one, and a
6811                // restore that did not work ends the chain: the call fails
6812                // and the caller's own (confirmed) restore decides.
6813                if let Some((wt, rev)) = restore
6814                    && let Err(e) = self.graft_restore(wt, rev).await
6815                {
6816                    let why =
6817                        format!("could not restore the worktree before the next fixer: {e:#}");
6818                    tracing::warn!("{why}");
6819                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6820                        AgentOutcome::ResourceBlocked(why)
6821                    } else {
6822                        AgentOutcome::Failed(why)
6823                    };
6824                    last = Some((job, seat, out));
6825                    break;
6826                }
6827                record_handover(
6828                    &mut self.state,
6829                    node,
6830                    &seat.key,
6831                    &spec.id,
6832                    next,
6833                    &class,
6834                    &reason,
6835                );
6836                continue;
6837            }
6838            last = Some((job, seat, out));
6839            break;
6840        }
6841        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6842        if let AgentOutcome::Quota(output) = &out
6843            && node != "graft"
6844        {
6845            // Graft is best effort: its caller records the loss and restores
6846            // the pre-graft tree, then proceeds to review and gate.
6847            // Every fallback was tried. Machine outages cannot be repaired by
6848            // spending a fix round; use each caller's resumable resource path.
6849            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6850            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6851            self.state.quota.push(loss);
6852            self.state.seats.insert(seat.key.clone(), seat.clone());
6853            self.state.status = RunStatus::Blocked;
6854            return (
6855                job,
6856                seat,
6857                AgentOutcome::ResourceBlocked(
6858                    "fixer unavailable; restore agent availability and resume".into(),
6859                ),
6860            );
6861        }
6862        (job, seat, out)
6863    }
6864
6865    /// The one repair a verification-only failure earns (see
6866    /// [`crate::recovery`]), before the first review round of a run that took
6867    /// a branch over.
6868    ///
6869    /// Saved as started *before* the fixer is called, so a resume never
6870    /// repairs twice; a start found on re-entry is rolled back to the commit
6871    /// it began from. A failed or unusable fixer leaves the tree as it was and
6872    /// the review goes on over it, so the first round's own verification
6873    /// reports the failure again. The failure is never carried into a round as
6874    /// a result: verification is re-run on whatever commit this leaves.
6875    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6876        use crate::recovery::{Mode, Repair};
6877        if !self.state.reviews.is_empty() {
6878            return Ok(());
6879        }
6880        let Some(mut rec) = self.state.recovery.clone() else {
6881            return Ok(());
6882        };
6883        if rec.mode != Mode::VerifyRepair {
6884            return Ok(());
6885        }
6886        let wt = winner.worktree.clone();
6887        let mut head = git::rev_parse(&wt, "HEAD").await?;
6888        match rec.repair.clone() {
6889            Repair::Done { .. } => return Ok(()),
6890            Repair::WaitingForRestore { before } => {
6891                self.graft_restore(&wt, &before).await?;
6892                head = before;
6893                rec.repair = Repair::NotStarted;
6894                self.state.recovery = Some(rec.clone());
6895                self.state.save()?;
6896            }
6897            Repair::Started { before } => {
6898                // Interrupted: the unfinished edits are not trusted.
6899                // An unconfirmed rollback blocks the run with `Started` kept,
6900                // so a resume retries it; the tree is never reviewed as is.
6901                if let Err(e) = self.graft_restore(&wt, &before).await {
6902                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6903                        return Err(e);
6904                    }
6905                    return self
6906                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6907                        .await;
6908                }
6909                rec.repair = Repair::Done {
6910                    before: before.clone(),
6911                    after: before,
6912                    committed: false,
6913                    failed: Some("interrupted; rolled back".to_owned()),
6914                };
6915                self.state
6916                    .event("recovery", "the repair was interrupted and rolled back");
6917                self.state.recovery = Some(rec);
6918                self.state.save()?;
6919                return Ok(());
6920            }
6921            Repair::NotStarted => {}
6922        }
6923        if crate::recovery::recheck(&mut rec, &head) {
6924            self.state.event("recovery", rec.summary());
6925            self.state.recovery = Some(rec);
6926            self.state.save()?;
6927            return Ok(());
6928        }
6929        rec.repair = Repair::Started {
6930            before: head.clone(),
6931        };
6932        self.state.recovery = Some(rec.clone());
6933        self.state.save()?;
6934
6935        let language = self.state.config.graph.language.clone();
6936        let sessions = self.state.config.graph.sessions;
6937        let artifacts = agent::artifacts_dir(&self.state.dir());
6938        let fix_prompt = prompt::recovery_fix(
6939            &self.state.instruction,
6940            &rec,
6941            self.state.config.graph.review_rounds,
6942            &language,
6943        );
6944        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6945        self.state.event(
6946            "recovery",
6947            format!(
6948                "repair: the fixer starts from the failed verification on {}",
6949                short(&head)
6950            ),
6951        );
6952        let (_job, seat, out) = self
6953            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6954                prompt: fix_prompt.clone(),
6955                spec,
6956                seat,
6957                cwd: wt.clone(),
6958                timeout,
6959                allow_write: true,
6960                sessions,
6961                artifacts: artifacts.clone(),
6962                stem: "fix-recovery".to_owned(),
6963                handover: None,
6964            })
6965            .await;
6966        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6967            rec.repair = Repair::WaitingForRestore { before: head };
6968            self.state.recovery = Some(rec);
6969            return self.stop_for_resource();
6970        }
6971        let failed = match &out {
6972            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6973            AgentOutcome::Quota(o) => {
6974                self.state.quota.push(QuotaLoss::from_report(
6975                    seat.key.clone(),
6976                    "fix",
6977                    o.quota.as_ref(),
6978                ));
6979                Some("rate limited (quota); the fixer could not run".to_owned())
6980            }
6981            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6982            AgentOutcome::Failed(e) => Some(e.clone()),
6983        };
6984        self.state.seats.insert(seat.key.clone(), seat);
6985        let coordination = self.coordinate(&wt).await?;
6986        let failed = match failed {
6987            // The agent's partial edits are not kept.
6988            Some(why) => {
6989                if let Err(e) = git::restore_to(&wt, &head).await {
6990                    return self
6991                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6992                        .await;
6993                }
6994                Some(why)
6995            }
6996            None => {
6997                if let Ok(r) = git::rescue_commit(
6998                    &wt,
6999                    "magi: recovery repair of the failed verification (uncommitted work)",
7000                )
7001                .await
7002                {
7003                    self.state.note_withheld("fix", &r.withheld);
7004                }
7005                None
7006            }
7007        };
7008        let after = git::rev_parse(&wt, "HEAD").await?;
7009        drop(coordination);
7010        self.state.event(
7011            "recovery",
7012            match &failed {
7013                Some(why) => format!("repair: nothing kept ({why})"),
7014                None if after != head => {
7015                    format!("repair: committed {}", short(&after))
7016                }
7017                None => "repair: the fixer made no commit".to_owned(),
7018            },
7019        );
7020        rec.repair = Repair::Done {
7021            before: head.clone(),
7022            committed: after != head,
7023            after,
7024            failed,
7025        };
7026        self.state.recovery = Some(rec);
7027        self.state.save()?;
7028        Ok(())
7029    }
7030
7031    /// A repair whose rollback could not be confirmed: the run is blocked and
7032    /// resumable, and the unconfirmed state is never reviewed.
7033    async fn stop_recovery(
7034        &mut self,
7035        mut rec: crate::recovery::RecoveryRecord,
7036        head: &str,
7037        why: String,
7038    ) -> Result<()> {
7039        // `Started` stays recorded: a resume retries the restore.
7040        rec.repair = crate::recovery::Repair::Started {
7041            before: head.to_owned(),
7042        };
7043        self.state.recovery = Some(rec);
7044        self.state
7045            .event("recovery", format!("repair could not be undone: {why}"));
7046        self.state.status = RunStatus::Blocked;
7047        self.state.save()?;
7048        bail!("recovery repair could not be rolled back: {why}")
7049    }
7050
7051    async fn review_loop(&mut self) -> Result<()> {
7052        if self.hold_base_sync().await? {
7053            return Ok(());
7054        }
7055        loop {
7056            if self
7057                .state
7058                .base_sync
7059                .as_ref()
7060                .is_some_and(|s| s.conflict.is_some())
7061            {
7062                return Ok(());
7063            }
7064            if self.catchup_repairable() {
7065                let record = self.state.reviews.last().expect("repairable round").clone();
7066                let winner = self
7067                    .state
7068                    .winner()
7069                    .cloned()
7070                    .context("catch-up repair has no winner")?;
7071                if !self.fix_review_round(&winner, record).await? {
7072                    return Ok(());
7073                }
7074            }
7075            self.review_loop_pass().await?;
7076            if !self.catchup_repairable() {
7077                return Ok(());
7078            }
7079        }
7080    }
7081
7082    /// Derive an owed catch-up repair from durable round evidence, including
7083    /// reentry after a crash before the fixer was dispatched.
7084    fn catchup_repairable(&self) -> bool {
7085        self.state.reviews.len() < self.state.config.graph.review_rounds
7086            && self.state.reviews.last().is_some_and(|r| {
7087                !r.clean
7088                    && r.blocking == 0
7089                    && r.fix.is_none()
7090                    && r.e2e_status() == E2eStatus::Failed
7091            })
7092    }
7093
7094    async fn review_loop_pass(&mut self) -> Result<()> {
7095        // A base that would not rebase is a person's decision, not a review
7096        // round: nothing here would change the answer, and reviewers and a
7097        // fixer would be spending real budget on a tree that cannot land
7098        // regardless of what they find.
7099        if self
7100            .state
7101            .base_sync
7102            .as_ref()
7103            .is_some_and(|s| s.conflict.is_some())
7104        {
7105            return Ok(());
7106        }
7107        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
7108        // agent files with `magi task add` name the run that paid for it. The
7109        // prompt overlay is cloned alongside it because the waves borrow it
7110        // while `self` is mutably borrowed by the node's own bookkeeping.
7111        let run_id = self.state.id.clone();
7112        let prompts = self.state.config.prompts.clone();
7113        let Some(winner) = self.state.winner().cloned() else {
7114            return Ok(());
7115        };
7116        let granted_review = self
7117            .state
7118            .repair_grants
7119            .last()
7120            .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7121            .cloned();
7122        let fresh_review = granted_review
7123            .as_ref()
7124            .is_some_and(|g| self.state.reviews.len() == g.review_start);
7125        let max_rounds = granted_review
7126            .as_ref()
7127            .map_or(self.state.config.graph.review_rounds, |g| {
7128                g.review_start.saturating_add(1)
7129            });
7130        // A clean round, an exhausted round budget, or a stalled tree (see
7131        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
7132        // they are recorded — recomputed here, not read off `status`, so a
7133        // reentry into a run that already stopped restates the identical
7134        // verdict instead of silently handing back whatever an earlier node
7135        // in this same walk clobbered `status` to (a solo-candidate
7136        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
7137        // below runs an empty range once the budget is spent, and would
7138        // otherwise fall through without touching `status` at all.
7139        if let Some(status) =
7140            review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7141        {
7142            // A reentry after a crash between the last round's save and
7143            // `stop_reviewing` reaches the hand-off here, not there.
7144            if status == RunStatus::Gating {
7145                self.record_contested_handoff();
7146            }
7147            self.state.status = status;
7148            self.state.save()?;
7149            return Ok(());
7150        }
7151        self.state.status = RunStatus::Reviewing;
7152        // A last recorded round whose own verification never resolved
7153        // (`ResourceBlocked` — the shared build cache, not the patch) is
7154        // never a concluded round, whatever the round budget says: starting
7155        // a fresh round on top of it would spend a whole new reviewer wave
7156        // re-reading an unchanged patch instead of just retrying the one
7157        // check that actually needs it, and once the budget is spent the
7158        // loop below has nothing left to do at all (its range is empty).
7159        // Retry that check directly instead, exactly the same retry
7160        // `stop_reviewing` already does for its own catch-up case.
7161        if !fresh_review
7162            && self.state.reviews.last().is_some_and(|r| {
7163                r.e2e_status() == E2eStatus::ResourceBlocked
7164                    || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7165            })
7166        {
7167            let shell = self.state.config.shell();
7168            return self
7169                .stop_unresolved_review(
7170                    "the last round's own verification never resolved",
7171                    &shell,
7172                    &winner.worktree,
7173                )
7174                .await;
7175        }
7176
7177        if granted_review.is_none() {
7178            self.recovery_repair(&winner).await?;
7179        }
7180        if self.resource_halt()? {
7181            return Ok(());
7182        }
7183
7184        let repo = self.state.repo.clone();
7185        let root = self.state.worktree_root();
7186        let language = self.state.config.graph.language.clone();
7187        let sessions = self.state.config.graph.sessions;
7188        let artifacts = self.merge_repair_artifacts();
7189        let base = self.landing_base();
7190        let base_short = short(&base);
7191        let reviewers = self.roles.reviewers.clone();
7192        let shell = self.state.config.shell();
7193
7194        for round in (self.state.reviews.len() + 1)..=max_rounds {
7195            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7196            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7197            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7198            // The prior round's own record, already persisted — never a
7199            // hand-carried variable of just its failing output: that is
7200            // exactly what let a round's e2e result drift out of sync with
7201            // which commit it was actually about (see `SCHEMA`'s doc for
7202            // schema 8). Judged against `head`, the commit reviewers are
7203            // about to look at now, so the summary always reads as "an
7204            // earlier head" here — this round's own patch has not been
7205            // checked yet.
7206            let prev_verification = self
7207                .state
7208                .reviews
7209                .last()
7210                .and_then(|r| r.verification_summary(&head));
7211            // Only the first round of a run that took a branch over: later
7212            // rounds have this run's own history.
7213            let prior_section = match (&self.state.recovery, round) {
7214                (Some(rec), 1) if self.state.reviews.is_empty() => {
7215                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7216                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7217                            Some(crate::recovery::head_of(
7218                                &git::diff(&winner.worktree, reviewed, "HEAD")
7219                                    .await
7220                                    .unwrap_or_default(),
7221                                crate::recovery::SINCE_DIFF_MAX,
7222                            ))
7223                        }
7224                        _ => None,
7225                    };
7226                    Some(prompt::recovery_review(rec, since.as_deref()))
7227                }
7228                _ => None,
7229            };
7230
7231            // Each reviewer gets its own detached checkout of exactly this
7232            // commit: nobody can perturb the winner's tree, and the fixer can
7233            // keep working without racing a reviewer.
7234            let mut jobs = Vec::new();
7235            let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7236                self.coordinate(&repo).await?
7237            } else {
7238                None
7239            };
7240            for (r, spec) in reviewers.iter().cloned().enumerate() {
7241                let wt = root.join(format!("review-{}", r + 1));
7242                // `clean -fdx` in a reset wipes setup's products, so every
7243                // round sets the seat up again.
7244                if wt.exists() {
7245                    crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7246                } else {
7247                    let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7248                        None
7249                    } else {
7250                        self.coordinate(&repo).await?
7251                    };
7252                    git::worktree_add_detached(&repo, &wt, &head).await?;
7253                }
7254                let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7255                    None
7256                } else {
7257                    self.coordinate(&repo).await?
7258                };
7259                self.setup_seat_worktree(&repo, &wt, false).await?;
7260                let seat_key = format!("review-{}", r + 1);
7261                // The seat starts the round on whoever answered it last, not
7262                // on the agent the spec names, so a failure is not re-paid.
7263                let spec = pick_start_spec(
7264                    &self.roles.reviewer_roster,
7265                    spec,
7266                    self.state.seat_history.get(&seat_key),
7267                );
7268                let seat = self.seat(&seat_key, &spec.id);
7269                jobs.push(SeatJob {
7270                    prompt: prompt::review(&prompt::ReviewCtx {
7271                        instruction: &self.state.instruction,
7272                        branch: &winner.branch,
7273                        base_short: &base_short,
7274                        stat: &stat,
7275                        patch: &patch,
7276                        verification: prev_verification.as_ref(),
7277                        prior: prior_section.as_deref(),
7278                        reviewers: reviewers.len(),
7279                        round,
7280                        rounds: max_rounds,
7281                        // A review-only run has no rankings, so nothing
7282                        // competed for this patch and the reviewer is told so.
7283                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7284                        lens: Lens::for_seat(r),
7285                        language: &language,
7286                        github_language: &self.state.config.graph.github_language,
7287                    }),
7288                    spec,
7289                    seat,
7290                    cwd: wt,
7291                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7292                    allow_write: false,
7293                    sessions,
7294                    artifacts: artifacts.clone(),
7295                    stem: format!("review-{round}-{}", r + 1),
7296                    handover: None,
7297                });
7298            }
7299
7300            drop(coordination);
7301            self.state.event(
7302                "review",
7303                format!(
7304                    "round {round}: {} reviewers on {}",
7305                    jobs.len(),
7306                    short(&head)
7307                ),
7308            );
7309            let mut quota_losses = Vec::new();
7310            let review_retries = self.state.config.graph.retries;
7311            let review_cache = self.state.config.cache_dir();
7312            let ctx = WaveCtx {
7313                carry_seats: true,
7314                pauses: Some((&self.pause, &self.interrupt)),
7315                run: &run_id,
7316                node: "review",
7317                prompts: &prompts,
7318                cache: review_cache.as_deref(),
7319                round: Some(round),
7320            };
7321            let results = ask_json_wave::<Review>(
7322                jobs,
7323                Arc::clone(&self.sem),
7324                review_retries,
7325                &self.roles.reviewer_roster,
7326                &ctx,
7327                &mut quota_losses,
7328                &mut self.state,
7329                &|_: &Review| Ok(()),
7330            )
7331            .await;
7332            if self.state.resource_block.is_some() {
7333                return self.stop_for_resource();
7334            }
7335            // Counted before the move below: how many of *this* round's
7336            // reviewer seats were lost to their own rate limit, as opposed to
7337            // a crash, a timeout, or unparsable output — see `round_is_clean`.
7338            let round_quota_missing = quota_losses.len();
7339            self.state.quota.extend(quota_losses);
7340
7341            let mut records = Vec::new();
7342            let mut all_findings = Vec::new();
7343            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7344                let agent_id = seat.agent.clone();
7345                self.state.seats.insert(seat.key.clone(), seat);
7346                let mut record = ReviewRecord {
7347                    reviewer: r + 1,
7348                    agent: agent_id,
7349                    summary: String::new(),
7350                    findings: Vec::new(),
7351                    vote: None,
7352                    failed: None,
7353                    duration_ms: 0,
7354                    // Set for both outcomes: `failed: Some(_)` with
7355                    // `attempts > 0` is a seat every retry still lost, not a
7356                    // recovered one — only `failed: None` with `attempts > 0`
7357                    // reads as "answered after a nudge" (see this field's own
7358                    // doc).
7359                    attempts,
7360                };
7361                match res {
7362                    Ok((review, out)) => {
7363                        // Sanitized here, at the point every other piece of
7364                        // agent prose in this file is (candidate summaries,
7365                        // deliberation turns, vote reasons): a reviewer's own
7366                        // words are the one thing about it that could name
7367                        // it, and reconsideration below broadcasts this same
7368                        // summary and these same findings to every other
7369                        // seat on the panel.
7370                        record.summary =
7371                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
7372                        record.vote = Some(review.vote);
7373                        record.duration_ms = out.duration_ms;
7374                        for (n, mut f) in review.findings.into_iter().enumerate() {
7375                            // ids are magi's, never the agent's: the fixer's
7376                            // adoption report is keyed by them.
7377                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7378                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7379                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7380                            // `file` is agent-supplied prose too, never
7381                            // checked against the real tree — the same
7382                            // exposure `title`/`detail` above have, just in
7383                            // a field easy to forget because it looks like a
7384                            // path rather than free text.
7385                            f.file = f
7386                                .file
7387                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7388                            all_findings.push(f.clone());
7389                            record.findings.push(f);
7390                        }
7391                        self.state.event(
7392                            "review",
7393                            format!(
7394                                "round {round}: reviewer {} voted {} with {} finding(s)",
7395                                r + 1,
7396                                review.vote.label(),
7397                                record.findings.len()
7398                            ),
7399                        );
7400                    }
7401                    Err(e) => {
7402                        record.failed = Some(e.to_string());
7403                        self.state.event(
7404                            "review",
7405                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7406                        );
7407                    }
7408                }
7409                records.push(record);
7410            }
7411
7412            // Tally the round's votes and, if they split, spend the one
7413            // round of reconsideration the split -> deliberate -> revote
7414            // shape `judge`/`vote` use for the panel, sized down to what a
7415            // read-only review round can afford: one round, and a revote
7416            // rather than an argument, because the panel already wrote its
7417            // reasoning down as findings the first time around.
7418            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7419            let vote_split =
7420                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7421            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7422            if vote_split {
7423                self.state.event(
7424                    "review",
7425                    format!(
7426                        "round {round}: votes split ({}) — one round of reconsideration",
7427                        initial_votes
7428                            .iter()
7429                            .map(|v| v.label())
7430                            .collect::<Vec<_>>()
7431                            .join(", ")
7432                    ),
7433                );
7434                // Seats read every seat's findings and votes, still numbered
7435                // and never named — the same anonymity `review` itself keeps.
7436                let panel: Vec<ReviewSeatReport<'_>> = records
7437                    .iter()
7438                    .filter_map(|r| {
7439                        r.vote.map(|vote| ReviewSeatReport {
7440                            reviewer: r.reviewer,
7441                            vote,
7442                            summary: &r.summary,
7443                            findings: &r.findings,
7444                        })
7445                    })
7446                    .collect();
7447
7448                let mut jobs = Vec::new();
7449                let mut seats_at = Vec::new();
7450                for (r, spec) in reviewers.iter().cloned().enumerate() {
7451                    // A seat with no initial vote has nothing to reconsider
7452                    // from and stays absent, the same as it stayed absent
7453                    // from `panel` above.
7454                    if records[r].vote.is_none() {
7455                        continue;
7456                    }
7457                    let wt = root.join(format!("review-{}", r + 1));
7458                    let seat_key = format!("review-{}", r + 1);
7459                    let spec = self.occupant(&seat_key, spec);
7460                    let seat = self.seat(&seat_key, &spec.id);
7461                    // A seat with no live session has already forgotten the
7462                    // initial review's prompt — restate the patch it is
7463                    // voting on, the same as `deliberate`/`vote` do for a
7464                    // judge in the same position.
7465                    // The panel already carries this seat's own review and
7466                    // vote, so restating the patch makes the prompt whole for
7467                    // a seat handed to another agent.
7468                    let build = |with_patch: bool| {
7469                        prompt::review_reconsider(&ReviewReconsiderCtx {
7470                            instruction: &self.state.instruction,
7471                            reviewer: r + 1,
7472                            lens: Lens::for_seat(r),
7473                            panel: &panel,
7474                            patch: with_patch.then_some(ReviewPatch {
7475                                branch: &winner.branch,
7476                                base_short: &base_short,
7477                                stat: &stat,
7478                                patch: &patch,
7479                            }),
7480                            round,
7481                            rounds: max_rounds,
7482                            language: &language,
7483                        })
7484                    };
7485                    let full = build(true);
7486                    let prompt = if has_context(&spec, &seat, sessions) {
7487                        build(false)
7488                    } else {
7489                        full.clone()
7490                    };
7491                    jobs.push(SeatJob {
7492                        prompt,
7493                        spec,
7494                        seat,
7495                        cwd: wt,
7496                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7497                        allow_write: false,
7498                        sessions,
7499                        artifacts: artifacts.clone(),
7500                        stem: format!("review-{round}-reconsider-{}", r + 1),
7501                        handover: Some(full),
7502                    });
7503                    seats_at.push(r);
7504                }
7505
7506                let mut recon_quota_losses = Vec::new();
7507                let recon_cache = self.state.config.cache_dir();
7508                let recon_ctx = WaveCtx {
7509                    carry_seats: true,
7510                    pauses: Some((&self.pause, &self.interrupt)),
7511                    run: &run_id,
7512                    node: "review",
7513                    prompts: &prompts,
7514                    cache: recon_cache.as_deref(),
7515                    round: Some(round),
7516                };
7517                let recon_results = ask_json_wave::<ReviewRevote>(
7518                    jobs,
7519                    Arc::clone(&self.sem),
7520                    review_retries,
7521                    &self.roles.reviewer_roster,
7522                    &recon_ctx,
7523                    &mut recon_quota_losses,
7524                    &mut self.state,
7525                    &|_: &ReviewRevote| Ok(()),
7526                )
7527                .await;
7528                if self.state.resource_block.is_some() {
7529                    return self.stop_for_resource();
7530                }
7531                self.state.quota.extend(recon_quota_losses);
7532
7533                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7534                    let agent_id = seat.agent.clone();
7535                    self.state.seats.insert(seat.key.clone(), seat);
7536                    let mut rec = ReviewRevoteRecord {
7537                        reviewer: r + 1,
7538                        agent: agent_id,
7539                        vote: None,
7540                        reason: String::new(),
7541                        failed: None,
7542                    };
7543                    match res {
7544                        Ok((rv, _)) => {
7545                            rec.vote = Some(rv.vote);
7546                            rec.reason =
7547                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7548                            self.state.event(
7549                                "review",
7550                                format!(
7551                                    "round {round}: reviewer {} revoted {}",
7552                                    r + 1,
7553                                    rv.vote.label()
7554                                ),
7555                            );
7556                        }
7557                        Err(e) => {
7558                            rec.failed = Some(e.to_string());
7559                            self.state.event(
7560                                "review",
7561                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7562                            );
7563                        }
7564                    }
7565                    reconsideration.push(rec);
7566                }
7567            } else if initial_votes.len() > 1 {
7568                self.state.event(
7569                    "review",
7570                    format!(
7571                        "round {round}: votes agreed ({}) — no reconsideration",
7572                        initial_votes[0].label()
7573                    ),
7574                );
7575            }
7576
7577            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7578            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7579            // A round that already has a blocking finding and a round left to
7580            // try is going back to the fixer no matter what `verify.e2e`
7581            // says, so running it first only spends the loop's slowest step
7582            // (minutes, for a Rust repo's full test suite) on a head about
7583            // to be rewritten. Deferred, never skipped: `verify.e2e` still
7584            // runs once a round has no blocking findings left (see
7585            // `round_is_clean`, which a deferred — empty — `e2e` can never
7586            // satisfy since `blocking` is nonzero whenever this branch is
7587            // taken), and `stop_reviewing` forces a real run before it will
7588            // ever read a deferred round as green.
7589            let defer_e2e =
7590                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7591            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7592                let reason =
7593                    format!("{blocking} blocking finding(s) already required a fix this round");
7594                self.state.event(
7595                    "verify",
7596                    format!(
7597                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7598                         {}); it will run once a round has none left",
7599                        short(&head)
7600                    ),
7601                );
7602                (Vec::new(), false, true, Some(reason))
7603            } else {
7604                let e2e_commands = self.state.config.verify.e2e.clone();
7605                let cache_dir = self.state.config.cache_dir();
7606                let context = format!("round {round}");
7607                let wait = self.repository_wait();
7608                let (e2e, verify_retried) = with_cache_lease(
7609                    &mut self.state,
7610                    cache_dir.as_deref(),
7611                    "e2e",
7612                    "e2e",
7613                    &winner.worktree,
7614                    &head,
7615                    verify_timeout,
7616                    &context,
7617                    &wait,
7618                    |state, budget| {
7619                        let shell = shell.clone();
7620                        let e2e_commands = e2e_commands.clone();
7621                        let worktree = winner.worktree.clone();
7622                        let context = context.clone();
7623                        async move {
7624                            run_e2e_with_retry(
7625                                state,
7626                                &shell,
7627                                &e2e_commands,
7628                                &worktree,
7629                                budget,
7630                                &context,
7631                            )
7632                            .await
7633                        }
7634                    },
7635                )
7636                .await?;
7637                (e2e, verify_retried, false, None)
7638            };
7639
7640            let expected = records.len();
7641            let answered = records.iter().filter(|r| r.failed.is_none()).count();
7642            let incomplete = answered < expected;
7643            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7644            let policy = self.state.config.graph.incomplete_review;
7645            let clean = round_is_clean(
7646                blocking,
7647                e2e_ok,
7648                answered,
7649                expected,
7650                round_quota_missing,
7651                policy,
7652            );
7653
7654            let mut round_record = ReviewRound {
7655                round,
7656                head: head.clone(),
7657                verified_head: None,
7658                verified_at: None,
7659                reviews: records,
7660                e2e,
7661                verify_retried,
7662                e2e_deferred,
7663                e2e_defer_reason,
7664                fix: None,
7665                blocking,
7666                answered,
7667                expected,
7668                clean,
7669                progressed: false,
7670                vote_split,
7671                reconsideration,
7672                verdict: None,
7673            };
7674            // The final vote per seat is its revote where reconsideration
7675            // ran and answered, its initial vote otherwise — the same
7676            // fallback `tally` uses for a judge whose private vote failed.
7677            round_record.verdict = ReviewVote::worst(
7678                round_record
7679                    .final_votes()
7680                    .into_iter()
7681                    .map(|(_, _, vote)| vote),
7682            );
7683            // Which commit and when magi actually attempted to check —
7684            // known the moment a command was dispatched against `head`,
7685            // whether or not it finished: a resource-blocked attempt still
7686            // targeted a specific commit at a specific time, and leaving
7687            // that unrecorded is exactly what made `verification_summary`
7688            // report a fresh attempt as "commit unknown ... recorded before
7689            // this was tracked", indistinguishable from a genuinely old,
7690            // untracked record. Only a deferred or unconfigured round never
7691            // ran at all and has nothing to record — see
7692            // `ReviewRound::verified_head`'s own doc.
7693            if !matches!(
7694                round_record.e2e_status(),
7695                E2eStatus::Deferred | E2eStatus::NotConfigured
7696            ) {
7697                round_record.verified_head = Some(head.clone());
7698                round_record.verified_at = Some(Timestamp::now());
7699            }
7700
7701            if incomplete {
7702                let missing: Vec<String> = round_record
7703                    .reviews
7704                    .iter()
7705                    .filter(|r| r.failed.is_some())
7706                    .map(|r| format!("review-{}", r.reviewer))
7707                    .collect();
7708                self.state.event(
7709                    "review",
7710                    format!(
7711                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7712                        missing.join(", ")
7713                    ),
7714                );
7715            }
7716
7717            if clean {
7718                self.state.event(
7719                    "review",
7720                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7721                        format!(
7722                            "round {round}: clean (warn policy, incomplete panel) — no \
7723                             blocking findings from the seats that answered, verification green"
7724                        )
7725                    } else if incomplete {
7726                        format!(
7727                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7728                             quorum) — no blocking findings from the seats that answered, \
7729                             verification green",
7730                            expected - answered
7731                        )
7732                    } else {
7733                        format!("round {round}: clean — no blocking findings, verification green")
7734                    },
7735                );
7736                self.state.reviews.push(round_record);
7737                self.state.status = RunStatus::Gating;
7738                self.state.save()?;
7739                return Ok(());
7740            }
7741
7742            // Nothing was raised and verification passed, but not every seat
7743            // answered and `round_is_clean` still refused to call it clean —
7744            // either a seat is missing for a reason other than its own quota
7745            // (a crash, a timeout, unparsable output — worth another try), or
7746            // every seat that could have answered lost its quota and nobody
7747            // is left to decide on: re-review rather than send the fixer
7748            // after a round with nothing to fix.
7749            if incomplete && blocking == 0 && e2e_ok {
7750                self.state.reviews.push(round_record);
7751                self.state.save()?;
7752                if round == max_rounds {
7753                    self.state.status = RunStatus::Blocked;
7754                    self.state.event(
7755                        "review",
7756                        format!(
7757                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7758                             refusing to call it clean",
7759                            expected - answered
7760                        ),
7761                    );
7762                    return Ok(());
7763                }
7764                continue;
7765            }
7766
7767            // Nothing for the fixer to act on (`blocking == 0`) and the only
7768            // reason this round is not clean is that magi itself never got
7769            // a command to run — the shared build cache, not the patch (see
7770            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7771            // the fixer would invite a change to appease contention that has
7772            // nothing to do with the diff, and would leave this attempt
7773            // sitting in the next round's prompt as if it were about an
7774            // earlier, superseded commit rather than what it actually is:
7775            // the same head, still waiting to be checked. Wait for it the
7776            // same way the final round's own contention is already handled,
7777            // whatever round this happens to be.
7778            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7779                self.state.reviews.push(round_record);
7780                return self
7781                    .stop_unresolved_review(
7782                        "the round's own verification could not run",
7783                        &shell,
7784                        &winner.worktree,
7785                    )
7786                    .await;
7787            }
7788
7789            if round == max_rounds {
7790                self.state.reviews.push(round_record);
7791                return self
7792                    .stop_reviewing(
7793                        &format!(
7794                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7795                        ),
7796                        &shell,
7797                        &winner.worktree,
7798                    )
7799                    .await;
7800            }
7801
7802            if !self.fix_review_round(&winner, round_record).await? {
7803                return Ok(());
7804            }
7805        }
7806        Ok(())
7807    }
7808
7809    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7810    /// False means admission or continuation policy stopped the review loop.
7811    async fn fix_review_round(
7812        &mut self,
7813        winner: &Candidate,
7814        mut round_record: ReviewRound,
7815    ) -> Result<bool> {
7816        let round = round_record.round;
7817        let shell = self.state.config.shell();
7818        let step = self.run_fix_round(winner, &mut round_record).await?;
7819        if matches!(step, FixStep::Resource) {
7820            self.stop_for_resource()?;
7821            return Ok(false);
7822        }
7823        if self.state.reviews.len() < round {
7824            self.state.reviews.push(round_record);
7825        } else {
7826            self.state.reviews[round - 1] = round_record;
7827        }
7828        self.state.save()?;
7829
7830        // The fixer's own report never came back this round, even after
7831        // `continue_fix_report`'s own budget was spent on it — not an
7832        // ordinary "no report" (dropped stream, quota, plain failure),
7833        // which already reads that way and is left to the existing round
7834        // budget. Stopping here, rather than opening another round, is
7835        // what keeps a next reviewer/fixer wave from ever being
7836        // dispatched onto `winner.worktree` while whatever the seat's
7837        // last call may still have running there is unaccounted for: no
7838        // process liveness check exists (and none is being added — see
7839        // AGENTS.md/this task's own scope), so the only way to honour
7840        // "nothing starts before a valid report returns" is to not start
7841        // anything further on this worktree from this run at all.
7842        if matches!(step, FixStep::Recorded { report_lost: true }) {
7843            self.stop_reviewing(
7844                "the fixer's adoption report never came back, even after resuming its \
7845                         own seat; refusing to start another round against the same worktree \
7846                         while that is unresolved",
7847                &shell,
7848                &winner.worktree,
7849            )
7850            .await?;
7851            return Ok(false);
7852        }
7853
7854        let streak = self
7855            .state
7856            .reviews
7857            .iter()
7858            .rev()
7859            .take_while(|r| !r.progressed)
7860            .count();
7861        if streak >= STAGNANT_LIMIT {
7862            self.stop_reviewing(
7863                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7864                &shell,
7865                &winner.worktree,
7866            )
7867            .await?;
7868            return Ok(false);
7869        }
7870        Ok(true)
7871    }
7872
7873    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7874    async fn run_fix_round(
7875        &mut self,
7876        winner: &Candidate,
7877        round_record: &mut ReviewRound,
7878    ) -> Result<FixStep> {
7879        let round = round_record.round;
7880        let max_rounds = self.state.config.graph.review_rounds;
7881        let run_id = self.state.id.clone();
7882        let prompts = self.state.config.prompts.clone();
7883        let language = self.state.config.graph.language.clone();
7884        let sessions = self.state.config.graph.sessions;
7885        let artifacts = self.merge_repair_artifacts();
7886        let base = self.landing_base();
7887        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7888        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7889        let this_round_verification = round_record.verification_summary(&head);
7890        let all_findings: Vec<_> = round_record
7891            .reviews
7892            .iter()
7893            .flat_map(|r| r.findings.iter().cloned())
7894            .collect();
7895        // Fix. The winner's own implementer seat continues its conversation:
7896        // the competition is over, so context is pure benefit now.
7897        let blocking_findings: Vec<_> = all_findings
7898            .iter()
7899            .filter(|f| f.severity.blocks())
7900            .cloned()
7901            .collect();
7902        let fix_prompt = prompt::fix_in(
7903            &self.state.instruction,
7904            &blocking_findings,
7905            this_round_verification.as_ref(),
7906            round,
7907            max_rounds,
7908            &language,
7909            &self.state.config.graph.github_language,
7910        );
7911        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7912        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7913        let (job, seat, out) = self
7914            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7915                prompt: fix_prompt.clone(),
7916                spec,
7917                seat,
7918                cwd: winner.worktree.clone(),
7919                timeout,
7920                allow_write: true,
7921                sessions,
7922                artifacts: artifacts.clone(),
7923                stem: format!("fix-{round}"),
7924                handover: None,
7925            })
7926            .await;
7927        // Not asked: the round is not recorded (so a resume repeats it),
7928        // no fixer was handed over to, and the seat is left as it was.
7929        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7930            return Ok(FixStep::Resource);
7931        }
7932        let agent_id = seat.agent.clone();
7933
7934        let mut fix = FixRecord {
7935            agent: agent_id,
7936            addressed: Vec::new(),
7937            rejected: Vec::new(),
7938            notes: String::new(),
7939            committed: false,
7940            failed: None,
7941            duration_ms: 0,
7942            continuation: None,
7943        };
7944        let mut continuation = ContinuationRecord::not_needed();
7945        let mut final_seat = seat.clone();
7946        match out {
7947            AgentOutcome::Ok(o) => {
7948                fix.duration_ms = o.duration_ms;
7949                let parsed = verdict::extract_json::<FixReport>(&o.text);
7950                // A parsed report standing next to a command this same
7951                // reply's own CLI never confirmed the exit status of is
7952                // not a resolved answer — the identical `CommandEvidence`
7953                // `state.jobs` renders, read here instead of only on
7954                // display, per the completion judgment and the shown
7955                // record needing to agree.
7956                let incomplete_reason = match &parsed {
7957                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7958                        "the reply parsed, but it reported a command whose own CLI never \
7959                             confirmed an exit status"
7960                            .to_owned(),
7961                    ),
7962                    Ok(_) => None,
7963                    Err(e) => Some(e.to_string()),
7964                };
7965                match incomplete_reason {
7966                    None => {
7967                        let report = parsed.expect("checked Ok above");
7968                        fix.addressed = report.addressed;
7969                        fix.rejected = report.rejected;
7970                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7971                    }
7972                    Some(reason) => {
7973                        let (resumed_seat, resolved, failure, cont) = self
7974                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7975                            .await;
7976                        fix.duration_ms += cont.cumulative_wait_ms;
7977                        continuation = cont;
7978                        final_seat = resumed_seat;
7979                        match resolved {
7980                            Some(report) => {
7981                                fix.addressed = report.addressed;
7982                                fix.rejected = report.rejected;
7983                                fix.notes =
7984                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
7985                            }
7986                            None => fix.failed = failure,
7987                        }
7988                    }
7989                }
7990            }
7991            // The CLI's raw error JSON is not a fix report to parse.
7992            AgentOutcome::Dropped(o) => {
7993                fix.duration_ms = o.duration_ms;
7994                let why = o
7995                    .dropped
7996                    .as_ref()
7997                    .map(|d| d.why.as_str())
7998                    .unwrap_or("the CLI ended the stream without delivering its answer");
7999                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
8000            }
8001            AgentOutcome::Quota(o) => {
8002                self.state.quota.push(QuotaLoss::from_report(
8003                    final_seat.key.clone(),
8004                    "fix",
8005                    o.quota.as_ref(),
8006                ));
8007                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
8008            }
8009            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8010        }
8011        if self.state.resource_block.is_some() {
8012            self.state.seats.insert(final_seat.key.clone(), final_seat);
8013            return Ok(FixStep::Resource);
8014        }
8015        fix.continuation = Some(continuation);
8016        self.state.seats.insert(final_seat.key.clone(), final_seat);
8017        let coordination = self.coordinate(&winner.worktree).await?;
8018        if let Ok(r) = git::rescue_commit(
8019            &winner.worktree,
8020            &format!("magi: review round {round} fixes (uncommitted work)"),
8021        )
8022        .await
8023        {
8024            self.state.note_withheld("fix", &r.withheld);
8025        }
8026        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8027        fix.committed = after != before;
8028        // Judged by what `git` says moved against base, never by the
8029        // fixer's own `addressed`/`rejected` count — see
8030        // `ReviewRound::progressed`. Propagated with `?`, the same as the
8031        // `patch` snapshot above: swallowing this error would default
8032        // `diff_after` to empty, which almost always differs from a
8033        // non-empty `patch` and reads as "progressed" — exactly backwards
8034        // for a `git` failure the stagnation check cannot see through.
8035        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8036        drop(coordination);
8037        let progressed = diff_after != patch;
8038        let commit_note = if fix.committed {
8039            "committed"
8040        } else {
8041            "NO new commit"
8042        };
8043        let tree_note = if progressed {
8044            "changed vs base"
8045        } else {
8046            "unchanged vs base"
8047        };
8048        self.state.event(
8049            "fix",
8050            match &fix.failed {
8051                // Distinct on purpose from "0 addressed, 0 rejected": the
8052                // fixer's own diff still landed (blocking counts do keep
8053                // falling round over round), only its adoption report did
8054                // not come back, so this must never read like every
8055                // finding was reviewed and declined.
8056                Some(reason) => {
8057                    format!(
8058                        "round {round}: fixer's adoption report was lost ({reason}); \
8059                             {commit_note}, tree {tree_note}"
8060                    )
8061                }
8062                None => format!(
8063                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8064                         {tree_note}{}",
8065                    fix.addressed.len(),
8066                    fix.rejected.len(),
8067                    if continuation.outcome == ContinuationOutcome::Resumed {
8068                        format!(
8069                            " (adoption report recovered after {} continuation(s))",
8070                            continuation.attempts
8071                        )
8072                    } else {
8073                        String::new()
8074                    },
8075                ),
8076            },
8077        );
8078        round_record.fix = Some(fix);
8079        round_record.progressed = progressed;
8080        Ok(FixStep::Recorded {
8081            report_lost: matches!(
8082                continuation.outcome,
8083                ContinuationOutcome::Exhausted
8084                    | ContinuationOutcome::QuotaLost
8085                    | ContinuationOutcome::NoSession
8086            ),
8087        })
8088    }
8089
8090    /// Decide, from the last recorded round's own verification, whether
8091    /// stopping the review loop is a hand-off or a genuine block.
8092    ///
8093    /// Called once the loop has given up trying — the round budget is spent,
8094    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
8095    /// findings still open, never while a round is still clean or the
8096    /// incomplete-panel case handled inline above. Gate and e2e are facts
8097    /// about the tree; a lingering review finding is an opinion, and this
8098    /// workload's own `magi stats` puts reviewer precision low enough
8099    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
8100    /// must not by itself stand between a green, verified change and the
8101    /// human who decides what to do with it. A red e2e is not an opinion, so
8102    /// that case still blocks, with the failing command and a tail of its
8103    /// output recorded here rather than left in `run.json` for someone to go
8104    /// find.
8105    ///
8106    /// A round that deferred its own e2e (see [`Config::graph`]'s
8107    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
8108    /// only because nothing ran, and treating an empty list as a passing one
8109    /// here is exactly the "deferred painted green" bug this function exists
8110    /// to not have. When the last round's own verification never resolved —
8111    /// deferred on purpose, or a real attempt the shared build cache blocked
8112    /// — this makes (or retries) the real run, on the actual worktree this
8113    /// loop is about to stop touching, before deciding anything. A
8114    /// resource-blocked attempt is likewise never read as either green or
8115    /// red: it is evidence about the machine, not the patch (see
8116    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
8117    /// blocked cache leaves this call without deciding rather than guessing
8118    /// — the caller retries on a later reentry.
8119    /// Record, once, that the review loop handed off over a blocking finding
8120    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
8121    /// `land` asks the owner even with `land_approval` off. Called from every
8122    /// path that concludes `Gating`; a reentry keeps the first record.
8123    fn record_contested_handoff(&mut self) {
8124        if self.state.contested_handoff.is_some() {
8125            return;
8126        }
8127        let Some(contested) = self
8128            .state
8129            .reviews
8130            .last()
8131            .and_then(ReviewRound::contested_handoff)
8132        else {
8133            return;
8134        };
8135        self.state.event(
8136            "review",
8137            format!(
8138                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8139                 wait for the owner's approval",
8140                contested.findings.len(),
8141                contested.rejecters.len()
8142            ),
8143        );
8144        self.state.contested_handoff = Some(contested);
8145    }
8146
8147    async fn stop_unresolved_review(
8148        &mut self,
8149        why: &str,
8150        shell: &[String],
8151        worktree: &Path,
8152    ) -> Result<()> {
8153        self.stop_reviewing_inner(why, shell, worktree, true).await
8154    }
8155
8156    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8157        self.stop_reviewing_inner(why, shell, worktree, false).await
8158    }
8159
8160    async fn stop_reviewing_inner(
8161        &mut self,
8162        why: &str,
8163        shell: &[String],
8164        worktree: &Path,
8165        repairable: bool,
8166    ) -> Result<()> {
8167        let round_idx = self.state.reviews.len() - 1;
8168        // A deferred round and a resource-blocked one are the same shape
8169        // here: neither has a real result yet, and both get one more
8170        // attempt. Read off `e2e_status` — the single source for this —
8171        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
8172        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
8173        // still retries instead of being read as a settled result the
8174        // instant it stops being empty.
8175        let needs_catchup_run = matches!(
8176            self.state.reviews[round_idx].e2e_status(),
8177            E2eStatus::Deferred | E2eStatus::ResourceBlocked
8178        );
8179        if needs_catchup_run {
8180            let round = self.state.reviews[round_idx].round;
8181            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8182            let commands = self.state.config.verify.e2e.clone();
8183            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8184            let cache_dir = self.state.config.cache_dir();
8185            let context = format!(
8186                "round {round}: verification unresolved, catching up before the final decision"
8187            );
8188            let wait = self.repository_wait();
8189            let (outcomes, verify_retried) = with_cache_lease(
8190                &mut self.state,
8191                cache_dir.as_deref(),
8192                "e2e",
8193                "e2e",
8194                worktree,
8195                &attempted_head,
8196                timeout,
8197                &context,
8198                &wait,
8199                |state, budget| {
8200                    let shell = shell.to_vec();
8201                    let commands = commands.clone();
8202                    let context = context.clone();
8203                    async move {
8204                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8205                            .await
8206                    }
8207                },
8208            )
8209            .await?;
8210            if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8211                self.state.record_base_progress(&attempted_head);
8212            }
8213            let last = &mut self.state.reviews[round_idx];
8214            last.e2e = outcomes;
8215            last.verify_retried = verify_retried;
8216            // Always the commit and time this attempt actually targeted,
8217            // whether or not it happens to equal the reviewed `head` and
8218            // whether or not a command finished — see
8219            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
8220            // attempt is recorded too, so a later reader sees "attempted
8221            // again at T2" rather than silence.
8222            last.verified_head = Some(attempted_head);
8223            last.verified_at = Some(Timestamp::now());
8224            if verify_inconclusive(&last.e2e) {
8225                // Still not a real result: `e2e_deferred` is left exactly
8226                // as it was, so `needs_catchup_run` above reads
8227                // `ResourceBlocked` (via `e2e_status`, which checks
8228                // `resource_blocked` before `e2e_deferred`) and retries
8229                // again on the next reentry, rather than recording
8230                // contention as a red e2e and blocking the run on it.
8231                self.state.save()?;
8232                return Ok(());
8233            }
8234            last.e2e_deferred = false;
8235            // A complete panel may have been saved as unclean solely because
8236            // verification could not run. Refresh that verdict with the new
8237            // outcomes before a granted repair consumes the round's result.
8238            // Incomplete panels retain their original policy/quota decision.
8239            if last.answered == last.expected {
8240                last.clean = round_is_clean(
8241                    last.blocking,
8242                    last.e2e.iter().all(CommandOutcome::ok),
8243                    last.answered,
8244                    last.expected,
8245                    0,
8246                    self.state.config.graph.incomplete_review,
8247                );
8248            }
8249        }
8250        let last = &self.state.reviews[round_idx];
8251        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8252
8253        match last.e2e_status() {
8254            E2eStatus::Failed => {
8255                let red: Vec<String> = last
8256                    .e2e
8257                    .iter()
8258                    .filter(|o| !o.ok())
8259                    .map(|o| {
8260                        format!(
8261                            "`{}` -> {:?}\n{}",
8262                            o.command,
8263                            o.code,
8264                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8265                        )
8266                    })
8267                    .collect();
8268                let can_repair = last.blocking == 0 && last.fix.is_none();
8269                self.state.reviews[round_idx].clean = false;
8270                self.state
8271                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8272                self.state.status = RunStatus::Blocked;
8273                if repairable
8274                    && can_repair
8275                    && self.state.reviews.len() < self.state.config.graph.review_rounds
8276                {
8277                    self.state.status = RunStatus::Reviewing;
8278                }
8279            }
8280            // `needs_catchup_run` above already retried once this call; if
8281            // it is still blocked, this is magi's own admission it could
8282            // not get a command to run, never a verdict on the patch — the
8283            // run is left exactly where a later reentry can retry again.
8284            E2eStatus::ResourceBlocked => {
8285                self.state.event(
8286                    "review",
8287                    format!(
8288                        "{why}; e2e could not run (resource unavailable); not \
8289                         deciding yet"
8290                    ),
8291                );
8292            }
8293            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8294                self.state.event(
8295                    "review",
8296                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8297                );
8298                self.record_contested_handoff();
8299                self.state.status = RunStatus::Gating;
8300            }
8301        }
8302        self.state.save()?;
8303        Ok(())
8304    }
8305
8306    // ----------------------------------------------------------------- gate
8307
8308    async fn gate(&mut self) -> Result<()> {
8309        if self.hold_base_sync().await? {
8310            return Ok(());
8311        }
8312        // Judged by the review record itself, not by `status`: a solo
8313        // candidate's `judge`/`deliberate` skip rewrites `status` on every
8314        // reentry (see `judge`), and trusting it here is exactly how a run
8315        // that exhausted its review budget got gated and merged a second
8316        // time around. `review_conclusion` recomputes the review loop's own
8317        // verdict from the round records themselves — `Gating` for a clean
8318        // round or a hand-off (see `stop_reviewing`), anything else means the
8319        // loop is still going or genuinely blocked.
8320        // A base the winner could not be replayed onto is a decision, not a
8321        // round: there is no landing tree to gate. Read as its own record for
8322        // the same reason the review verdict is.
8323        if self.state.status == RunStatus::Failed
8324            || self
8325                .state
8326                .base_sync
8327                .as_ref()
8328                .is_some_and(|s| s.conflict.is_some())
8329            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8330                != Some(RunStatus::Gating)
8331        {
8332            return Ok(());
8333        }
8334        let Some(winner) = self.state.winner().cloned() else {
8335            return Ok(());
8336        };
8337        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8338        self.state.reconcile_verification_head(&head);
8339        self.state.save()?;
8340        let wait = self.repository_wait();
8341        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8342            return Ok(());
8343        }
8344        if self.state.gate_ran {
8345            // `review_loop` derives its conclusion from the clean review
8346            // record on every reentry and therefore puts a completed run back
8347            // in `Gating`. A recorded gate is a stronger, terminal fact:
8348            // retain its original command output (or lack of any, for a repo
8349            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
8350            // doc) and restore `Blocked` on a real failure rather than
8351            // pretending the command is still running or running it a second
8352            // time. `gate_ran == false` remains the only shape — unattempted,
8353            // or a resource-blocked retry — that may still need to execute a
8354            // command.
8355            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8356                self.state.status = RunStatus::Blocked;
8357                self.state.save()?;
8358            }
8359            return Ok(());
8360        }
8361        self.state.status = RunStatus::Gating;
8362        let mut outcomes = self.run_gate(&winner).await?;
8363        let mut attempted_head = self.state.gate_head.clone();
8364        loop {
8365            // A resource-blocked outcome means the gate command never actually
8366            // ran - the shared build cache could not be acquired or confirmed
8367            // fresh in time - which is evidence about the machine, not about
8368            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
8369            // Recording it as a red gate would mark a run `Blocked` on nothing
8370            // but contention magi has already logged; leaving `self.state.gate`
8371            // empty and `self.state.gate_ran` false instead keeps the shape
8372            // this function already treats as "still needs to run" (see the
8373            // early-return above), so the next call retries the command
8374            // rather than concluding anything.
8375            if verify_inconclusive(&outcomes) {
8376                self.state.save()?;
8377                return Ok(());
8378            }
8379            if outcomes.iter().all(CommandOutcome::ok) {
8380                break;
8381            }
8382            let fixes_before = self.state.gate_fixes.len();
8383            match self.gate_fix_round(&winner, &outcomes).await? {
8384                GateFix::Retry => {
8385                    outcomes = self.run_gate(&winner).await?;
8386                    attempted_head = self.state.gate_head.clone();
8387                }
8388                GateFix::Stop => {
8389                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8390                    if self.state.gate_fixes.len() > fixes_before
8391                        && attempted_head.as_deref() != Some(current.as_str())
8392                        && verify_current_head(&mut self.state, &winner, &wait).await?
8393                    {
8394                        // A new SHA owes a gate even when a spent fixer changed only
8395                        // commit metadata. This does not grant another repair round.
8396                        outcomes = self.run_gate(&winner).await?;
8397                        attempted_head = self.state.gate_head.clone();
8398                    } else {
8399                        break;
8400                    }
8401                }
8402                GateFix::Defer => {
8403                    self.state.save()?;
8404                    return Ok(());
8405                }
8406            }
8407        }
8408        let passed = outcomes.iter().all(CommandOutcome::ok);
8409        self.state.gate = outcomes;
8410        self.state.gate_ran = true;
8411        self.state.gate_head = attempted_head;
8412        if !self.state.gate.is_empty()
8413            && !verify_inconclusive(&self.state.gate)
8414            && let Some(head) = self.state.gate_head.clone()
8415        {
8416            self.state.record_base_progress(&head);
8417        }
8418        self.state.save()?;
8419        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8420            return Ok(());
8421        }
8422        if !passed {
8423            self.state.status = RunStatus::Blocked;
8424            let spent = self.state.gate_fixes.len();
8425            self.state.event(
8426                "gate",
8427                if spent == 0 {
8428                    "gate failed; not merging".to_owned()
8429                } else {
8430                    format!("gate failed after {spent} gate-fix round(s); not merging")
8431                },
8432            );
8433        }
8434        self.state.save()?;
8435        Ok(())
8436    }
8437
8438    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
8439    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
8440    /// after review is clean and never on a candidate awaiting judging.
8441    ///
8442    /// Never fails the run: a non-zero exit or timeout is a warning and a
8443    /// recorded outcome, and the gate remains the single arbiter. Nothing
8444    /// configured means nothing happens - no event, no commit. `commit_all`
8445    /// commits any leftover change under the neutral identity and returns
8446    /// `false` when the tree is clean, so no empty commit is ever made.
8447    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8448        let commands = state.config.verify.pre_gate.clone();
8449        if commands.is_empty() {
8450            return;
8451        }
8452        let shell = state.config.shell();
8453        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8454        let (outcomes, _) = run_commands(
8455            state,
8456            "pre_gate",
8457            "pre_gate",
8458            0,
8459            &shell,
8460            &commands,
8461            &winner.worktree,
8462            timeout,
8463        )
8464        .await;
8465        for o in &outcomes {
8466            if !o.ok() {
8467                tracing::warn!(
8468                    "pre_gate `{}` failed ({:?}); the gate decides",
8469                    o.command,
8470                    o.code
8471                );
8472            }
8473            state.event(
8474                "pre_gate",
8475                format!(
8476                    "`{}` -> {}",
8477                    o.command,
8478                    if o.ok() {
8479                        "pass".to_owned()
8480                    } else {
8481                        format!(
8482                            "FAIL ({:?})\n{}",
8483                            o.code,
8484                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8485                        )
8486                    }
8487                ),
8488            );
8489        }
8490        state.pre_gate = outcomes;
8491        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8492            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8493                Ok(head) => {
8494                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8495                    state.pre_gate_commit = Some(head);
8496                }
8497                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8498            },
8499            Ok(false) => {}
8500            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8501        }
8502        if let Err(e) = state.save() {
8503            tracing::warn!("could not persist the pre_gate record: {e:#}");
8504        }
8505    }
8506
8507    /// Run `verify.gate` once against the winner's current tree, logging one
8508    /// event per command. Empty when nothing is configured.
8509    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8510        let wait = self.repository_wait();
8511        run_landing_gate(&mut self.state, winner, &wait).await
8512    }
8513
8514    /// One bounded fix round for a failing gate.
8515    ///
8516    /// The fixer is told the failure came from the gate itself, not from a
8517    /// reviewer, and is shown the failed commands, their exit codes and a tail
8518    /// of their output - whatever `[verify].gate` holds, nothing here knows
8519    /// what those commands run. Only a normal non-zero exit that printed
8520    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
8521    /// command or a full disk says nothing about the code, and a fixer sent
8522    /// after it can only appease the machine. The round is judged by what git
8523    /// says moved, never by the fixer's own report, and `verify.e2e` runs
8524    /// again before the gate does, so a fix cannot trade a green gate for a
8525    /// red e2e unnoticed.
8526    async fn gate_fix_round(
8527        &mut self,
8528        winner: &Candidate,
8529        outcomes: &[CommandOutcome],
8530    ) -> Result<GateFix> {
8531        let cap = if self.state.repair_grants.last().is_some_and(|g| {
8532            g.phase == crate::repair_budget::Phase::Fixing
8533                && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8534        }) {
8535            self.state.gate_fixes.len().saturating_add(1)
8536        } else {
8537            self.state.config.graph.gate_fix_rounds
8538        };
8539        let spent = self.state.gate_fixes.len();
8540        if spent >= cap {
8541            if cap > 0 {
8542                self.state.event(
8543                    "gate",
8544                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
8545                );
8546            }
8547            return Ok(GateFix::Stop);
8548        }
8549        if !gate_fixable(outcomes) {
8550            self.state.event(
8551                "gate",
8552                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8553                 command or similar); not spending a fix round on it",
8554            );
8555            return Ok(GateFix::Stop);
8556        }
8557        let min_free = self.state.config.disk.min_free_bytes;
8558        if min_free > 0 {
8559            match crate::disk::free_bytes(&winner.worktree) {
8560                Ok(free) if crate::disk::enough_space(free, min_free) => {}
8561                Ok(free) => {
8562                    self.state.event(
8563                        "gate",
8564                        format!(
8565                            "only {free} bytes free ({min_free} required by `[disk] \
8566                             min_free_bytes`); not spending a fix round on a failure the disk \
8567                             may explain"
8568                        ),
8569                    );
8570                    return Ok(GateFix::Stop);
8571                }
8572                Err(e) => {
8573                    self.state.event(
8574                        "gate",
8575                        format!("free disk space could not be measured ({e:#}); no fix round"),
8576                    );
8577                    return Ok(GateFix::Stop);
8578                }
8579            }
8580        }
8581
8582        let attempt = spent + 1;
8583        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8584        let base = self.landing_base();
8585        let fix_prompt = prompt::gate_fix_in(
8586            &self.state.instruction,
8587            &failed,
8588            attempt,
8589            cap,
8590            &self.state.config.graph.language,
8591            &self.state.config.graph.github_language,
8592        );
8593        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8594        let sessions = self.state.config.graph.sessions;
8595        let artifacts = self.merge_repair_artifacts();
8596        self.state.event(
8597            "gate",
8598            format!("gate failed; gate-fix round {attempt} of {cap}"),
8599        );
8600        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8601        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8602        // Persist before invoking the fixer: it may commit and then be
8603        // interrupted before its result (or the following e2e) is saved.
8604        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8605            head: before.clone(),
8606            passed: false,
8607            outcomes: Vec::new(),
8608        });
8609        self.state.save()?;
8610        let (_, seat, out) = self
8611            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8612                prompt: fix_prompt.clone(),
8613                spec,
8614                seat,
8615                cwd: winner.worktree.clone(),
8616                timeout,
8617                allow_write: true,
8618                sessions,
8619                artifacts: artifacts.clone(),
8620                stem: format!("gate-fix-{attempt}"),
8621                handover: None,
8622            })
8623            .await;
8624        // The fixer was never asked: no round is spent (nothing is recorded)
8625        // and a later reentry asks the same agent again.
8626        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8627            return Ok(GateFix::Defer);
8628        }
8629        let mut record = GateFixRecord {
8630            agent: seat.agent.clone(),
8631            failed,
8632            notes: String::new(),
8633            committed: false,
8634            error: None,
8635        };
8636        match out {
8637            AgentOutcome::Ok(o) => {
8638                // A missing report is not a failed fix: the round is judged
8639                // by the tree below, and the report only carries prose.
8640                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8641                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8642                }
8643            }
8644            AgentOutcome::Dropped(_) => {
8645                record.error = Some("the CLI dropped the stream".to_owned());
8646            }
8647            AgentOutcome::Quota(o) => {
8648                self.state.quota.push(QuotaLoss::from_report(
8649                    seat.key.clone(),
8650                    "gate-fix",
8651                    o.quota.as_ref(),
8652                ));
8653                record.error = Some("rate limited (quota); fixer could not run".to_owned());
8654            }
8655            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8656        }
8657        self.state.seats.insert(seat.key.clone(), seat);
8658        let coordination = self.coordinate(&winner.worktree).await?;
8659        if let Ok(r) = git::rescue_commit(
8660            &winner.worktree,
8661            &format!("magi: gate fix {attempt} (uncommitted work)"),
8662        )
8663        .await
8664        {
8665            self.state.note_withheld("gate-fix", &r.withheld);
8666        }
8667        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8668        record.committed = after != before;
8669        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8670        let note = record.error.clone();
8671        self.state.gate_fixes.push(record);
8672        self.state.save()?;
8673        if !changed {
8674            self.state.event(
8675                "gate",
8676                match note {
8677                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8678                    None => format!("gate-fix round {attempt}: the tree did not change"),
8679                },
8680            );
8681            return Ok(GateFix::Stop);
8682        }
8683        self.state.event(
8684            "gate",
8685            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8686        );
8687
8688        drop(coordination);
8689        let wait = self.repository_wait();
8690        check_gate_repair_e2e(&mut self.state, winner, &wait).await
8691    }
8692
8693    // ---------------------------------------------------------------- merge
8694
8695    /// One read-only rewrite by the summary's author; a text the gate still
8696    /// withholds goes to the owner (see `github_text`) and the fixed fallback
8697    /// applies on their word or on silence. `None` means the run is parked on
8698    /// that question and nothing may be pushed or opened yet.
8699    async fn guarded_pr_message(
8700        &mut self,
8701        winner: &Candidate,
8702        facts: Option<&BranchFacts>,
8703        posting: bool,
8704    ) -> Result<Option<PrMessage>> {
8705        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8706        let head = match final_head.as_deref() {
8707            Some(h) => crate::pr_outcome::FinalHead::At(h),
8708            None => crate::pr_outcome::FinalHead::Unknown,
8709        };
8710        // Only a pull request that is about to be posted pays for a summary.
8711        let summarized = if posting {
8712            Some(self.summarize_change(winner, final_head.as_deref()).await)
8713        } else {
8714            None
8715        };
8716        // A busy shared resource is not a permanent cause: publishing an
8717        // "unavailable" description would stick (a later head move never
8718        // edits the body). Stop before anything is pushed or opened; the
8719        // run is `Blocked` and a resume asks the summary again.
8720        if self.state.resource_block.is_some() {
8721            self.state.event(
8722                "pr-summary",
8723                "a shared resource was busy; nothing was published, resume to summarize again",
8724            );
8725            return Ok(None);
8726        }
8727        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8728            anyhow::bail!(
8729                "the final commit moved during public description generation; retry before publishing"
8730            );
8731        }
8732        let note = match (&summarized, &self.state.change_summary) {
8733            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8734            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8735            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8736            (None, _) => ChangeNote::NotAsked,
8737        };
8738        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8739        if !posting {
8740            let identity = crate::scrub::Identity::current();
8741            return Ok(Some(PrMessage {
8742                title: crate::scrub::scrub(&pr.title, &identity),
8743                body: crate::scrub::scrub(&pr.body, &identity),
8744            }));
8745        }
8746        // This very text was already rejected and asked about: no second
8747        // judge call, rewrite or question, just carry the record on.
8748        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8749        if let Some(known) = self
8750            .state
8751            .github_text
8752            .clone()
8753            .filter(|g| g.fingerprint == raw_fp)
8754        {
8755            return self.settle_withheld(winner, &pr, known).await;
8756        }
8757        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8758        let mut violations =
8759            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8760        let identity = crate::scrub::Identity::current();
8761        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8762            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8763            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8764        {
8765            violations.push(crate::github_text::Violation::SensitiveData);
8766        }
8767        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8768        // only a language violation starts one; sensitive data alone goes
8769        // straight to the owner, with the redacted text as the fallback.
8770        let has_language = violations
8771            .iter()
8772            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8773        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8774            self.state.event(
8775                "github-text",
8776                if has_language {
8777                    format!("description rejected: {violations:?}; requesting one rewrite")
8778                } else {
8779                    format!("description rejected: {violations:?}; asking the owner")
8780                },
8781            );
8782            let mut rewritten = false;
8783            let mut rejected_rewrite = None;
8784            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8785            {
8786                let output = text;
8787                if let Some(title) = summary_title(&output) {
8788                    // Judge the redacted candidate that will actually be posted.
8789                    // Sensitive data alone must not discard a useful rewrite.
8790                    let raw_body = summary_without_title(&output);
8791                    let clean_title = crate::scrub::scrub(&title, &identity);
8792                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8793                    let redacted = clean_title != title || body != raw_body;
8794                    let title = clean_title;
8795                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8796                    if !complete {
8797                        self.state.event(
8798                            "github-text",
8799                            "description rewrite omitted required findings",
8800                        );
8801                    }
8802                    if !body.trim().is_empty() {
8803                        body.push_str("\n\n");
8804                        body.push_str(&public_pr_evidence(&self.state, head));
8805                    }
8806                    let marker = format!("magi:run/{}", self.state.id);
8807                    if !body.trim().is_empty() && !body.contains(&marker) {
8808                        body.push_str(&format!("\n\n{marker}"));
8809                    }
8810                    let rewrite = PrMessage {
8811                        title: title.clone(),
8812                        body: body.clone(),
8813                    };
8814                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8815                    let mut failed = crate::github_text::check_for(
8816                        &self.state.config.graph,
8817                        &title,
8818                        &body,
8819                        redecision,
8820                    );
8821                    if (!crate::github_text::shareable(&title)
8822                        || !crate::github_text::shareable(&body))
8823                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8824                    {
8825                        failed.push(crate::github_text::Violation::SensitiveData);
8826                    }
8827                    if complete
8828                        && !title.trim().is_empty()
8829                        && !body.trim().is_empty()
8830                        && !failed.is_empty()
8831                    {
8832                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8833                            title: title.clone(),
8834                            body: body.clone(),
8835                            failed: failed
8836                                .iter()
8837                                .map(|v| crate::github_text::category(*v).to_owned())
8838                                .collect(),
8839                        });
8840                    }
8841                    if complete
8842                        && !title.trim().is_empty()
8843                        && !raw_body.trim().is_empty()
8844                        && failed.is_empty()
8845                    {
8846                        decision = redecision;
8847                        pr = PrMessage { title, body };
8848                        rewritten = true;
8849                        self.state
8850                            .event("github-text", "description rewrite passed");
8851                        if redacted {
8852                            self.state.event(
8853                                "github-text",
8854                                "sensitive data removed from description rewrite",
8855                            );
8856                        }
8857                    }
8858                }
8859            }
8860            if !rewritten {
8861                self.state.event(
8862                    "github-text",
8863                    "description rewrite unavailable or rejected; asking the owner",
8864                );
8865                if let Some(w) =
8866                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8867                {
8868                    // A different text than the one asked about before: that
8869                    // question no longer describes anything.
8870                    self.retire_github_text_question("the withheld text changed");
8871                    let record = crate::run::GithubTextAsk {
8872                        rewrite: rejected_rewrite,
8873                        fingerprint: raw_fp,
8874                        title: w.title,
8875                        body: w.body,
8876                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8877                        question: None,
8878                        asks: 0,
8879                        resolved: false,
8880                        chosen_title: None,
8881                        choice: None,
8882                        publication: None,
8883                    };
8884                    return self.settle_withheld(winner, &pr, record).await;
8885                }
8886            }
8887        }
8888        let (title, body) =
8889            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8890        Ok(Some(PrMessage { title, body }))
8891    }
8892
8893    /// Describe the whole change from its final diff in either run mode.
8894    ///
8895    /// The summary is tied to the final commit and to the merge base of the
8896    /// resolved remote base; a record for any other pair is dropped and the
8897    /// agent asked again, so a description never outlives the diff it was
8898    /// written from. The seat is read-only and the prompt tells it to read the
8899    /// diff and the code around it. `[roles] synthesizer` selects the agents
8900    /// (unset: the default pick), each asked at most once, advancing on an
8901    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8902    /// carries the reason it is unavailable; nothing is invented then.
8903    async fn summarize_change(
8904        &mut self,
8905        winner: &Candidate,
8906        head: Option<&str>,
8907    ) -> std::result::Result<(), String> {
8908        let Some(head) = head else {
8909            return Err("the final commit could not be read".to_owned());
8910        };
8911        let remote = self.state.config.merge.remote.clone();
8912        let tracking = format!("{remote}/{}", self.state.base_branch);
8913        if !git::rev_exists(&winner.worktree, &tracking).await {
8914            return Err(format!("the remote base `{tracking}` could not be read"));
8915        }
8916        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8917            Ok(b) if !b.is_empty() => b,
8918            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8919        };
8920        let evidence = crate::pr_outcome::summary_evidence(
8921            &self.state,
8922            crate::pr_outcome::FinalHead::At(head),
8923        );
8924        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8925        if self
8926            .state
8927            .change_summary
8928            .as_ref()
8929            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8930        {
8931            return Ok(());
8932        }
8933        if self.state.change_summary.take().is_some() {
8934            self.state.event(
8935                "pr-summary",
8936                "the final diff or evidence changed; the earlier change summary was discarded",
8937            );
8938        }
8939        // Save the source snapshot even if no generation agent is available.
8940        if let Err(e) = write_artifact(
8941            &self.state,
8942            &format!("public-description-{fingerprint}.md"),
8943            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8944        ) {
8945            tracing::warn!("could not save public description evidence: {e:#}");
8946        }
8947        let chain = match agent::pick_chain(
8948            &self.state.config.agents,
8949            self.state.config.roles.synthesizer.as_ref(),
8950            &agent::installed,
8951            "synthesizer",
8952        ) {
8953            Ok(chain) => chain,
8954            Err(e) => {
8955                tracing::warn!("no agent for the change summary: {e:#}");
8956                self.state.event("pr-summary", "no summary agent available");
8957                return Err("no agent was available to summarize it".to_owned());
8958            }
8959        };
8960        let subjects = git::subjects(&winner.worktree, &base, head)
8961            .await
8962            .unwrap_or_default();
8963        let stat = git::diff_stat(&winner.worktree, &base, head)
8964            .await
8965            .unwrap_or_default();
8966        let stat: String = stat.chars().take(6_000).collect();
8967        let mut prompt = crate::prompt::change_summary_in(
8968            &base,
8969            head,
8970            &subjects,
8971            stat.trim(),
8972            &self.state.config.graph.github_language,
8973        );
8974        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8975        let sessions = self.state.config.graph.sessions;
8976        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8977        let prompts = self.state.config.prompts.clone();
8978        let cache = self.state.config.cache_dir();
8979        let run = self.state.id.clone();
8980        for (i, spec) in chain.into_iter().enumerate() {
8981            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8982            let github_spec = match agent::for_github_text(
8983                &spec,
8984                &self.state.config.graph.github_language,
8985                &winner.worktree,
8986            ) {
8987                Ok(spec) => spec,
8988                Err(e) => {
8989                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8990                    continue;
8991                }
8992            };
8993            let job = SeatJob {
8994                spec: github_spec,
8995                seat,
8996                cwd: winner.worktree.clone(),
8997                prompt: prompt.clone(),
8998                timeout,
8999                allow_write: false,
9000                sessions,
9001                artifacts: agent::artifacts_dir(&self.state.dir()),
9002                stem: if i == 0 {
9003                    "change-summary".to_owned()
9004                } else {
9005                    format!("change-summary-{}", spec.id)
9006                },
9007                handover: None,
9008            };
9009            let ctx = WaveCtx {
9010                run: &run,
9011                node: "pr-summary",
9012                prompts: &prompts,
9013                cache: cache.as_deref(),
9014                round: None,
9015                carry_seats: false,
9016                pauses: Some((&self.pause, &self.interrupt)),
9017            };
9018            let (seat, outcome) =
9019                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9020            self.state.seats.insert(seat.key.clone(), seat);
9021            let text = match outcome {
9022                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9023                AgentOutcome::ResourceBlocked(why) => {
9024                    // No agent was asked: the run stops resumably before
9025                    // anything is published (see `guarded_pr_message`).
9026                    if self.state.resource_block.is_none() {
9027                        self.state.resource_block = Some(crate::run::ResourceBlock {
9028                            cause: crate::run::ResourceBlockCause::Other,
9029                            node: "pr-summary".to_owned(),
9030                            kind: "shared resource".to_owned(),
9031                            detail: why,
9032                            at: Timestamp::now(),
9033                        });
9034                    }
9035                    return Err("a shared resource was busy".to_owned());
9036                }
9037                _ => continue,
9038            };
9039            let blind = &self.state.config.blind;
9040            let Some(title) = summary_title(&text) else {
9041                continue;
9042            };
9043            let body: String = summary_without_title(&text)
9044                .lines()
9045                .filter(|l| !l.contains("magi:run/"))
9046                .collect::<Vec<_>>()
9047                .join("\n");
9048            let body = blind::sanitize_prose(body.trim(), blind);
9049            let title = blind::sanitize_prose(&title, blind);
9050            if body.trim().is_empty() || title.trim().is_empty() {
9051                continue;
9052            }
9053            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9054                continue;
9055            }
9056            // The branch must still be the one the agent was shown.
9057            if git::rev_parse(&winner.worktree, "HEAD")
9058                .await
9059                .ok()
9060                .as_deref()
9061                != Some(head)
9062            {
9063                return Err("the branch moved while the summary was written".to_owned());
9064            }
9065            if git::merge_base(&winner.worktree, &tracking, head)
9066                .await
9067                .ok()
9068                .as_deref()
9069                != Some(base.as_str())
9070                || crate::pr_outcome::summary_evidence(
9071                    &self.state,
9072                    crate::pr_outcome::FinalHead::At(head),
9073                ) != evidence
9074            {
9075                return Err("the summary evidence changed while it was written".to_owned());
9076            }
9077            self.state.change_summary = Some(crate::run::ChangeSummary {
9078                head: head.to_owned(),
9079                base,
9080                evidence: fingerprint,
9081                title,
9082                body,
9083            });
9084            self.state.event(
9085                "pr-summary",
9086                format!("change summary written by {} for the final diff", spec.id),
9087            );
9088            if let Err(e) = self.state.save() {
9089                tracing::warn!("could not save the change summary: {e:#}");
9090            }
9091            return Ok(());
9092        }
9093        self.state
9094            .event("pr-summary", "no agent produced a change summary");
9095        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9096    }
9097
9098    /// One read-only rewrite request for a withheld description. The
9099    /// implementer's own conversation is tried first when it exists and can be
9100    /// resumed; then the `[roles] language_judge` chain (or the default pick)
9101    /// writes it in a separate conversation, skipping the implementer's agent.
9102    /// The prompt carries the whole text, so no agent needs any memory. Each
9103    /// agent is asked at most once and the chain advances on an error, quota,
9104    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
9105    /// that has run out of quota must not end the rewrite. Nothing is recorded
9106    /// as a handover.
9107    async fn rewrite_pr_text(
9108        &mut self,
9109        winner: &Candidate,
9110        pr: &PrMessage,
9111        violations: &[crate::github_text::Violation],
9112    ) -> Option<String> {
9113        let prompt = format!(
9114            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9115            pr.title, pr.body
9116        );
9117        let sessions = self.state.config.graph.sessions;
9118        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9119        // (spec, the resumed implementer's seat). A fallback seat is created
9120        // only when its turn comes, so an unused fallback leaves no record.
9121        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9122        let key = format!("impl-{}", winner.label);
9123        if let Some(spec) = self
9124            .state
9125            .config
9126            .agents
9127            .iter()
9128            .find(|a| a.id == winner.agent)
9129            .cloned()
9130            && let Some(existing) = self.state.seats.get(&key)
9131            && existing.agent == spec.id
9132            && agent::has_session(spec.kind, existing, sessions)
9133        {
9134            attempts.push((spec, Some(existing.clone())));
9135        }
9136        match agent::pick_chain(
9137            &self.state.config.agents,
9138            self.state.config.roles.language_judge.as_ref(),
9139            &agent::installed,
9140            "language_judge",
9141        ) {
9142            Ok(chain) => {
9143                for spec in chain {
9144                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9145                        continue;
9146                    }
9147                    attempts.push((spec, None));
9148                }
9149            }
9150            Err(e) if attempts.is_empty() => {
9151                tracing::warn!("no agent for the description rewrite: {e:#}");
9152                self.state
9153                    .event("github-text", "no rewrite agent available");
9154                return None;
9155            }
9156            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9157        }
9158        let prompts = self.state.config.prompts.clone();
9159        let cache = self.state.config.cache_dir();
9160        let run = self.state.id.clone();
9161        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9162            let resumed = resumed_seat.is_some();
9163            let seat = resumed_seat.unwrap_or_else(|| {
9164                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9165            });
9166            let github_spec = match agent::for_github_text(
9167                &spec,
9168                &self.state.config.graph.github_language,
9169                &winner.worktree,
9170            ) {
9171                Ok(spec) => spec,
9172                Err(e) => {
9173                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9174                    continue;
9175                }
9176            };
9177            let job = SeatJob {
9178                spec: github_spec,
9179                seat,
9180                cwd: winner.worktree.clone(),
9181                prompt: prompt.clone(),
9182                timeout: retry_budget(timeout_full, resumed),
9183                allow_write: false,
9184                sessions,
9185                artifacts: agent::artifacts_dir(&self.state.dir()),
9186                stem: if i == 0 {
9187                    "github-text-rewrite".to_owned()
9188                } else {
9189                    format!("github-text-rewrite-{}", spec.id)
9190                },
9191                handover: None,
9192            };
9193            let ctx = WaveCtx {
9194                run: &run,
9195                node: "github-text",
9196                prompts: &prompts,
9197                cache: cache.as_deref(),
9198                round: None,
9199                carry_seats: false,
9200                pauses: Some((&self.pause, &self.interrupt)),
9201            };
9202            let (seat, outcome) =
9203                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9204            self.state.seats.insert(seat.key.clone(), seat);
9205            match outcome {
9206                AgentOutcome::Ok(o)
9207                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9208                {
9209                    return Some(o.text);
9210                }
9211                AgentOutcome::ResourceBlocked(_) => return None,
9212                _ => {}
9213            }
9214        }
9215        None
9216    }
9217
9218    /// Abandon the standing posting-gate question, if any.
9219    fn retire_github_text_question(&mut self, why: &str) {
9220        let Some(id) = self
9221            .state
9222            .github_text
9223            .as_ref()
9224            .and_then(|g| g.question.clone())
9225        else {
9226            return;
9227        };
9228        let _ = ask::Questions::open().update(&id, |q| {
9229            q.abandon(why);
9230            Ok(())
9231        });
9232    }
9233
9234    /// Carry a withheld text to its end: file the question, park while it is
9235    /// open, and on an answer, silence or timeout pick the text to post. The
9236    /// decision is saved before it is used, so it applies once.
9237    async fn settle_withheld(
9238        &mut self,
9239        winner: &Candidate,
9240        pr: &PrMessage,
9241        mut g: crate::run::GithubTextAsk,
9242    ) -> Result<Option<PrMessage>> {
9243        use crate::github_text as gt;
9244        let store = ask::Questions::open();
9245        let timeout = self.state.config.graph.answer_timeout;
9246        if !g.resolved {
9247            let mut standing = g
9248                .question
9249                .as_ref()
9250                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9251            if let Some(q) = standing.as_ref()
9252                && q.status.open()
9253                && gt::expired(q, timeout)
9254            {
9255                let why = format!("no answer within {}s of asking", timeout.max(1));
9256                standing = store
9257                    .update(&q.id, |q| {
9258                        q.abandon(&why);
9259                        Ok(())
9260                    })
9261                    .ok()
9262                    .map(|(q, ())| q);
9263            }
9264            let mut ask_again: Option<Option<Vec<String>>> = None;
9265            match standing {
9266                // Recorded but never written (a stop between the two saves),
9267                // or removed: ask again without spending the retry.
9268                None if g.asks <= 1 => {
9269                    g.asks = 0;
9270                    ask_again = Some(None);
9271                }
9272                None => {
9273                    g.resolved = true;
9274                    g.chosen_title = None;
9275                    g.choice = Some(GithubTextChoice::Fallback);
9276                }
9277                Some(q) if q.status.open() => {
9278                    self.park_on_github_text(&g)?;
9279                    return Ok(None);
9280                }
9281                Some(q) => match gt::read_reply(&q) {
9282                    gt::Reply::Fallback => {
9283                        g.resolved = true;
9284                        g.chosen_title = None;
9285                        g.choice = Some(GithubTextChoice::Fallback);
9286                    }
9287                    gt::Reply::Original => {
9288                        g.resolved = true;
9289                        g.chosen_title = None;
9290                        g.choice = Some(GithubTextChoice::Original);
9291                    }
9292                    gt::Reply::Rewrite => {
9293                        g.resolved = true;
9294                        g.chosen_title = None;
9295                        g.choice = Some(if g.rewrite.is_some() {
9296                            GithubTextChoice::Rewrite
9297                        } else {
9298                            GithubTextChoice::Fallback
9299                        });
9300                    }
9301                    gt::Reply::Title(text) => {
9302                        let first = text
9303                            .lines()
9304                            .map(str::trim)
9305                            .find(|l| !l.is_empty())
9306                            .unwrap_or("");
9307                        let decision = if first.is_empty() {
9308                            None
9309                        } else {
9310                            let candidate = PrMessage {
9311                                title: first.to_owned(),
9312                                body: String::new(),
9313                            };
9314                            self.judge_pr_language(&winner.worktree, &candidate).await
9315                        };
9316                        match gt::vet_title(
9317                            &text,
9318                            decision,
9319                            self.state.config.graph.github_english(),
9320                        ) {
9321                            Ok(title) => {
9322                                g.resolved = true;
9323                                g.chosen_title = Some(title);
9324                                g.choice = Some(GithubTextChoice::MyTitle);
9325                            }
9326                            Err(cats) if g.asks < 2 => {
9327                                ask_again = Some(Some(
9328                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9329                                ))
9330                            }
9331                            Err(_) => {
9332                                g.resolved = true;
9333                                g.chosen_title = None;
9334                                g.choice = Some(GithubTextChoice::Fallback);
9335                            }
9336                        }
9337                    }
9338                },
9339            }
9340            if let Some(rejected) = ask_again {
9341                // A retry (after a rejected replacement title) is still about
9342                // the original text: the same ORIGINAL / FALLBACK comparison.
9343                let w = gt::Withheld {
9344                    title: g.title,
9345                    body: g.body,
9346                    categories: g.categories.clone(),
9347                };
9348                // Named by the original text's fingerprint alone, so a resume
9349                // rewrites the same file and the detail stays identical.
9350                let artifact = match crate::run::write_artifact(
9351                    &self.state,
9352                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
9353                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9354                ) {
9355                    Ok(p) => Some(p),
9356                    Err(e) => {
9357                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
9358                        None
9359                    }
9360                };
9361                let (detail, comparison) = gt::question_content(
9362                    &w,
9363                    &pr.title,
9364                    &pr.body,
9365                    &self.state.id,
9366                    rejected.as_deref(),
9367                    artifact.as_deref(),
9368                    g.rewrite.as_ref(),
9369                );
9370                let mut q = ask::Question::new(
9371                    self.state.id.clone(),
9372                    gt::ASK_NODE.to_owned(),
9373                    gt::ASK_SEAT.to_owned(),
9374                    gt::question_summary(&self.state.config.graph.language, &w),
9375                    detail,
9376                    gt::question_choices(&w, g.rewrite.as_ref()),
9377                );
9378                q.posting_comparison = Some(comparison);
9379                q.answer_timeout = timeout;
9380                // Recorded before it is written: a stop in between leaves a
9381                // record naming a question the resume then files again.
9382                g.question = Some(q.id.clone());
9383                g.asks += 1;
9384                self.state.github_text = Some(g.clone());
9385                self.state.event(
9386                    "github-text",
9387                    format!("asking the owner about the withheld text ({})", q.short()),
9388                );
9389                self.state.save()?;
9390                store
9391                    .put_executable(&mut q)
9392                    .context("file the posting-gate question")?;
9393                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9394                    tracing::warn!(
9395                        "could not notify about posting-gate question {}: {e:#}",
9396                        q.short()
9397                    );
9398                }
9399                self.park_on_github_text(&g)?;
9400                return Ok(None);
9401            }
9402        }
9403        let resolved = g.chosen_title.clone();
9404        let original = g.choice == Some(GithubTextChoice::Original);
9405        // Field names only: the event log never records the text.
9406        self.state.event(
9407            "github-text",
9408            if original {
9409                format!(
9410                    "posting the original text for: {}",
9411                    match (g.title, g.body) {
9412                        (true, true) => "title, description",
9413                        (true, false) => "title",
9414                        _ => "description",
9415                    }
9416                )
9417            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9418                "posting the saved rewrite title and description".to_owned()
9419            } else if resolved.is_some() {
9420                "posting the owner's replacement title".to_owned()
9421            } else {
9422                "posting the neutral text for the withheld fields".to_owned()
9423            },
9424        );
9425        self.state.github_text = Some(g.clone());
9426        self.state.save()?;
9427        if g.choice == Some(GithubTextChoice::Rewrite)
9428            && let Some(r) = g.rewrite.as_ref()
9429        {
9430            return Ok(Some(PrMessage {
9431                title: r.title.clone(),
9432                body: r.body.clone(),
9433            }));
9434        }
9435        let w = gt::Withheld {
9436            title: g.title,
9437            body: g.body,
9438            categories: g.categories.clone(),
9439        };
9440        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9441        if let Some(t) = resolved {
9442            title = t;
9443        }
9444        // Every withheld field is now neutral or vetted English.
9445        let english = gt::LanguageDecision {
9446            title_english: true,
9447            body_english: true,
9448            body_complete: true,
9449        };
9450        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9451        if original {
9452            // The owner's explicit choice overrides the gate for the fields it
9453            // was made for; the others keep what the gate produced.
9454            if g.title {
9455                title = pr.title.clone();
9456            }
9457            if g.body {
9458                body = pr.body.clone();
9459            }
9460        }
9461        Ok(Some(PrMessage { title, body }))
9462    }
9463
9464    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9465        self.state.github_text = Some(g.clone());
9466        self.state.end_pass_parked(ParkCause::Question);
9467        self.state.parked = true;
9468        self.state.event(
9469            "github-text",
9470            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9471        );
9472        self.state.save()
9473    }
9474
9475    /// Ask `[roles] language_judge` about this text and record which source
9476    /// decided; the text itself is never recorded.
9477    async fn judge_pr_language(
9478        &mut self,
9479        cwd: &Path,
9480        pr: &PrMessage,
9481    ) -> Option<crate::github_text::LanguageDecision> {
9482        self.state.config.roles.language_judge.as_ref()?;
9483        if !self.state.config.graph.language_gate() {
9484            return None;
9485        }
9486        let decision =
9487            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9488        self.state.event(
9489            "github-text",
9490            if decision.is_some() {
9491                "language decided by the language judge"
9492            } else {
9493                "language judge unavailable; using built-in heuristics"
9494            },
9495        );
9496        decision
9497    }
9498
9499    fn merge(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9500        Box::pin(self.merge_inner())
9501    }
9502
9503    async fn merge_inner(&mut self) -> Result<()> {
9504        if self.resource_halt()? || self.hold_base_sync().await? {
9505            return Ok(());
9506        }
9507        // Same reasoning as `gate`: ask the review and gate records directly
9508        // rather than `status`, which a solo-candidate `judge`/`deliberate`
9509        // skip can rewrite on reentry to something that no longer says
9510        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
9511        // so a hand-off (open findings, green verification) reaches merge
9512        // exactly like a genuinely clean round does.
9513        //
9514        // A run resumed mid-`land` never reaches here at all: `execute`
9515        // recognises `RunStatus::Landing` before it even calls `prep`, and
9516        // routes straight to `run_land` instead. That has to happen a level
9517        // up from this function, not with a check in here, because
9518        // `review_loop`'s own status recomputation (see its doc) runs
9519        // *before* `merge` on every reentry and would otherwise overwrite
9520        // the `Landing` marker with `Gating` before this node ever saw it.
9521        if self
9522            .state
9523            .base_sync
9524            .as_ref()
9525            .is_some_and(|s| s.conflict.is_some())
9526            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9527                != Some(RunStatus::Gating)
9528            // `gate_ran == false` is not "passed" - `gate` leaves it false
9529            // both before it has ever run and when its last attempt was
9530            // resource-blocked (see `Runner::gate`'s own doc), and neither is
9531            // permission to merge on nothing but the review record. Only a
9532            // gate that actually ran may proceed; an intentional local
9533            // review can also finish with verification explicitly not configured.
9534            // `RunState::gate_status` computes that distinction.
9535            || !self.state.gate_allows_progress()
9536        {
9537            return Ok(());
9538        }
9539        // This node's own record, not `status`: `status == Ready` is not
9540        // unique to the harmless `MergeMode::None` path this line was
9541        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
9542        // run's PR was closed without merging — and on that run `mode` is
9543        // still `Pr`, so a reentry that fell through here would push and
9544        // open a second pull request. `self.state.merge` is set exactly once
9545        // this node (or `land`) has already produced a verdict, under every
9546        // mode, which is what "already done" actually means here.
9547        if self.state.merge.is_some() && !self.state.pr_publication_pending {
9548            return Ok(());
9549        }
9550        let Some(winner) = self.state.winner().cloned() else {
9551            return Ok(());
9552        };
9553        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9554        if self.state.gate_head.as_deref() != Some(head.as_str()) {
9555            self.state.status = RunStatus::Blocked;
9556            self.state.event(
9557                "gate",
9558                "winner changed after gate; fresh verification required",
9559            );
9560            self.state.save()?;
9561            return Ok(());
9562        }
9563        let wait = self.repository_wait();
9564        if !verify_current_head(&mut self.state, &winner, &wait).await? {
9565            return Ok(());
9566        }
9567        let repo = self.state.repo.clone();
9568        let base = self.state.base_branch.clone();
9569        let mode = self.state.config.merge.mode;
9570        let style = self.state.config.merge.style;
9571        let facts = if is_review_run(&self.state) {
9572            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9573            let start = review_base(
9574                &repo,
9575                &self.state.config.merge.remote,
9576                &base,
9577                &self.state.base_commit,
9578                &winner.branch,
9579            )
9580            .await;
9581            branch_facts(&repo, &start, &winner.branch).await
9582        } else {
9583            None
9584        };
9585        // `None` when the base could not be freshly read: then an adopted
9586        // pull request's title is left alone.
9587        let leaked = if is_review_run(&self.state) {
9588            leaked_subjects(&self.state, &winner.branch).await
9589        } else {
9590            Some(Vec::new())
9591        };
9592        // `None`: parked on the owner's word about a withheld title or body.
9593        // Nothing has been pushed or opened, and `merge` stays unrecorded so
9594        // the resume arrives here again.
9595        let Some(pr) = self
9596            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9597            .await?
9598        else {
9599            return Ok(());
9600        };
9601        let message = pr.commit_message();
9602
9603        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9604        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9605            return Ok(());
9606        }
9607        let outcome = match mode {
9608            MergeMode::None => MergeOutcome {
9609                mode,
9610                ok: true,
9611                detail: manual_merge_command(style, &repo, &winner.branch, &message),
9612                empty: false,
9613            },
9614            MergeMode::Pr | MergeMode::Local
9615                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9616            {
9617                MergeOutcome {
9618                    mode,
9619                    ok: false,
9620                    detail: empty_candidate_detail(&self.state, &base),
9621                    empty: true,
9622                }
9623            }
9624            MergeMode::Local => {
9625                let on = git::current_branch(&repo).await?;
9626                if on.as_deref() != Some(base.as_str()) {
9627                    MergeOutcome {
9628                        mode,
9629                        ok: false,
9630                        detail: format!(
9631                            "{} has {} checked out, not the base branch {base}",
9632                            repo.display(),
9633                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
9634                        ),
9635                        empty: false,
9636                    }
9637                } else if !git::is_clean(&repo).await? {
9638                    MergeOutcome {
9639                        mode,
9640                        ok: false,
9641                        detail: format!("{} is dirty; refusing to merge", repo.display()),
9642                        empty: false,
9643                    }
9644                } else {
9645                    let out = match style {
9646                        MergeStyle::Merge => {
9647                            git::merge_no_ff(&repo, &winner.branch, &message).await?
9648                        }
9649                        MergeStyle::Squash => {
9650                            git::merge_squash(&repo, &winner.branch, &message).await?
9651                        }
9652                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9653                    };
9654                    MergeOutcome {
9655                        mode,
9656                        ok: out.ok(),
9657                        detail: if out.ok() { out.stdout } else { out.stderr },
9658                        empty: false,
9659                    }
9660                }
9661            }
9662            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9663                self.state.pr_publication_pending = true;
9664                self.state.save()?;
9665                let out =
9666                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9667                self.state.pr_publication_pending = out.is_err();
9668                MergeOutcome {
9669                    mode,
9670                    ok: out.is_ok(),
9671                    detail: match out {
9672                        Ok(url) => url,
9673                        Err(e) => format!("{e:#}"),
9674                    },
9675                    empty: false,
9676                }
9677            }
9678            MergeMode::Pr => {
9679                let remote = self.state.config.merge.remote.clone();
9680                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9681                let pushed = if self.state.foreign_checkout.is_some() {
9682                    let refspec = format!("{delivery_head}:refs/heads/{public}");
9683                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9684                } else {
9685                    git::push(&winner.worktree, &remote, &winner.branch).await?
9686                };
9687                if !pushed.ok() {
9688                    MergeOutcome {
9689                        mode,
9690                        ok: false,
9691                        detail: pushed.stderr,
9692                        empty: false,
9693                    }
9694                } else {
9695                    if self.state.foreign_checkout.is_some() {
9696                        self.state.pr_pushed_head = Some(delivery_head.clone());
9697                    }
9698                    // A retry or resume of a run whose branch already has an
9699                    // open pull request adopts it rather than failing on a
9700                    // duplicate. Only this winner branch into this base:
9701                    // `branch_for` derives the name from the run id, so a
9702                    // different run's pull request never matches.
9703                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9704                    let out = match pr_merge_plan(found) {
9705                        PrPlan::Create => {
9706                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9707                                .await
9708                        }
9709                        PrPlan::Adopt { url, title } => {
9710                            self.state
9711                                .event("merge", format!("Pr: adopted open pull request {url}"));
9712                            if self.state.explicit_github_text_selection() {
9713                                // An explicit publication selection applies the prepared
9714                                // payload; acceptance alone is not publication proof.
9715                                land::set_pr_text_verbatim(
9716                                    &winner.worktree,
9717                                    &url,
9718                                    &pr.title,
9719                                    &pr.body,
9720                                )
9721                                .await
9722                                .map(|()| url)
9723                            } else {
9724                                if title != pr.title
9725                                    && (!is_review_run(&self.state)
9726                                        || leaked
9727                                            .as_deref()
9728                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9729                                    && let Err(e) =
9730                                        if self.state.github_text.as_ref().is_some_and(|g| {
9731                                            g.resolved
9732                                                && g.title
9733                                                && g.choice == Some(GithubTextChoice::Original)
9734                                        }) {
9735                                            // The owner chose to post this title unmodified.
9736                                            land::set_pr_title_verbatim(
9737                                                &winner.worktree,
9738                                                &url,
9739                                                &pr.title,
9740                                            )
9741                                            .await
9742                                        } else {
9743                                            land::set_pr_title(
9744                                                &mut self.state,
9745                                                &winner.worktree,
9746                                                &url,
9747                                                &pr.title,
9748                                            )
9749                                            .await
9750                                        }
9751                                {
9752                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9753                                    self.state
9754                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9755                                }
9756                                Ok(url)
9757                            }
9758                        }
9759                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9760                    };
9761                    match out {
9762                        Ok(url) => MergeOutcome {
9763                            mode,
9764                            ok: true,
9765                            detail: url,
9766                            empty: false,
9767                        },
9768                        Err(e) => MergeOutcome {
9769                            mode,
9770                            ok: false,
9771                            detail: e.to_string(),
9772                            empty: false,
9773                        },
9774                    }
9775                }
9776            }
9777        };
9778
9779        self.state.status = match (mode, outcome.ok) {
9780            (MergeMode::None, _) => RunStatus::Ready,
9781            (_, true) if outcome.empty => RunStatus::Ready,
9782            (MergeMode::Pr, true) => RunStatus::Ready,
9783            (_, true) => RunStatus::Merged,
9784            (_, false) => RunStatus::Blocked,
9785        };
9786        self.state.event(
9787            "merge",
9788            format!(
9789                "{:?}: {}",
9790                mode,
9791                outcome.detail.lines().next().unwrap_or("")
9792            ),
9793        );
9794        if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9795            self.state.record_github_text_publication(&outcome.detail);
9796        }
9797        self.state.merge = Some(outcome);
9798        self.state.save()?;
9799
9800        // The PR is open and the run would historically stop here, leaving the
9801        // operator to watch checks, feed review comments back to a fixer, and
9802        // merge. That was done by hand six times in one session before this
9803        // existed. Opt-in, because merging is the one irreversible thing magi
9804        // can do to a repository.
9805        if self.state.config.graph.land
9806            && mode == MergeMode::Pr
9807            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9808        {
9809            self.run_land().await?;
9810        }
9811        // `run_land` may have left `status` at `Landing` - still waiting on
9812        // CI or the owner's approval, not actually settled - so this has to
9813        // read whatever `status` ended up as here, not the `Merged` this
9814        // function set a few lines up.
9815        self.settle_questions();
9816        Ok(())
9817    }
9818
9819    /// Enter `land`.
9820    ///
9821    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9822    /// a resumed run's re-entry. `land::land` itself is what serialises the
9823    /// two git-mutating moments inside the loop — the rebase push and
9824    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9825    /// to hold a lock across the whole call, and doing so would serialise
9826    /// this run's CI wait against a *different* run's land-approval resume
9827    /// in the same repository, which is exactly the "must not wait on
9828    /// another task" property the daemon's slot-freeing exists to give.
9829    fn run_land(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9830        Box::pin(self.run_land_inner())
9831    }
9832
9833    async fn run_land_inner(&mut self) -> Result<()> {
9834        crate::pr_head::ensure_owner(&self.state)?;
9835        let url = self
9836            .state
9837            .pr
9838            .as_ref()
9839            .map(|pr| pr.url.clone())
9840            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9841            .unwrap_or_default();
9842        let url = url.lines().next().unwrap_or("").trim().to_owned();
9843        if !url.starts_with("http") {
9844            return Ok(());
9845        }
9846        // A land failure is not a lost run: the work is on a branch and the
9847        // pull request is open, which is exactly where a human takes over.
9848        loop {
9849            match land::land(&mut self.state, &url).await {
9850                Ok(_)
9851                    if self
9852                        .state
9853                        .merge_repairs
9854                        .last()
9855                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9856                {
9857                    if self.repair_merge_review().await? {
9858                        continue;
9859                    }
9860                }
9861                Ok(_) if self.state.resource_block.is_some() => {
9862                    return self.stop_for_resource();
9863                }
9864                Ok(pr) if self.state.parked => {
9865                    // `land` already saved the parked marker; nothing here
9866                    // overrides `status` back to a terminal value while an
9867                    // approval is still outstanding.
9868                    let _ = pr;
9869                }
9870                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9871                    self.state.status = RunStatus::Superseded;
9872                    self.state.save()?;
9873                }
9874                Ok(pr) => {
9875                    self.state.status = match pr.state {
9876                        land::PrLifecycle::Merged => RunStatus::Merged,
9877                        _ => RunStatus::Blocked,
9878                    };
9879                    // Downstream of a confirmed merge only - see
9880                    // `bump::should_release_bump`'s own doc for why this one
9881                    // check covers all three of `land`'s success paths.
9882                    // Best-effort: the run already landed, so a failure here
9883                    // (the decision call, `gh`, `cargo`) is recorded and never
9884                    // turns a landed run into a failed one.
9885                    if bump::should_release_bump(self.state.status)
9886                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9887                    {
9888                        // The event is the run's own record. Not-eligible cases
9889                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9890                        // `Err` is a bump that was tried and failed:
9891                        // `after_merge` itself raises the operator notice for
9892                        // that, whether or not a release PR exists yet.
9893                        self.state
9894                            .event("bump", format!("release bump skipped: {e:#}"));
9895                    }
9896                    // Independent of the bump, and best-effort in the same way:
9897                    // findings the merge left open become follow-up tasks.
9898                    if self.state.status == RunStatus::Merged {
9899                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9900                    }
9901                    self.state.save()?;
9902                }
9903                Err(e) => {
9904                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9905                        return Err(e);
9906                    }
9907                    self.state.status = RunStatus::Blocked;
9908                    self.state.event("land", format!("gave up: {e}"));
9909                    self.state.save()?;
9910                }
9911            }
9912            break;
9913        }
9914        Ok(())
9915    }
9916
9917    /// One repair cycle stays under the existing driver/task claim. An
9918    /// interrupted fixer is rolled back and held rather than run twice.
9919    async fn repair_merge_review(&mut self) -> Result<bool> {
9920        use crate::run::MergeRepairPhase as Phase;
9921        crate::pr_head::ensure_owner(&self.state)?;
9922        let i = self.state.merge_repairs.len() - 1;
9923        let request = self.state.merge_repairs[i].clone();
9924        if request.phase == Phase::Fixing {
9925            let restored = match self.state.winner().cloned() {
9926                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9927                None => Err(anyhow::anyhow!("repair has no winning branch")),
9928            };
9929            let why = match restored {
9930                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9931                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9932                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9933            };
9934            self.hold_merge_repair(i, why)?;
9935            return Ok(false);
9936        }
9937        // A completed repair gate is a publication checkpoint. On reentry,
9938        // reject external changes before the ordinary gate path can replace
9939        // that evidence by verifying a new head. Requested repairs still
9940        // start their own review generation below.
9941        if request.phase == Phase::Reviewing && self.state.gate_ran {
9942            let winner = self
9943                .state
9944                .winner()
9945                .context("repair has no winning branch")?;
9946            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9947            if self.state.gate_head.as_deref() != Some(head.as_str())
9948                || !git::is_clean(&winner.worktree).await?
9949            {
9950                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9951                return Ok(false);
9952            }
9953        }
9954        let failure = if i >= self.state.config.graph.land_rounds
9955            || self.state.config.graph.review_rounds == 0
9956        {
9957            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9958        } else {
9959            None
9960        };
9961        if let Some(why) = failure {
9962            self.hold_merge_repair(i, why)?;
9963            return Ok(false);
9964        }
9965        if request.phase == Phase::Requested {
9966            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9967                Ok(pr) => pr,
9968                Err(e) => {
9969                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9970                    return Ok(false);
9971                }
9972            };
9973            self.state.merge_repairs[i].phase = Phase::Fixing;
9974            self.state.land_approval = None;
9975            self.state.land_armed_head = None;
9976            self.state.save()?;
9977            // Serialize the exact records, including finding ids/details,
9978            // reconsideration reasons, final votes and verification output.
9979            let evidence = serde_json::to_string_pretty(&request)?;
9980            let reason = format!(
9981                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9982            );
9983            let budget = self.state.config.graph.land_rounds;
9984            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9985            match result {
9986                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9987                Ok(land::Fixed::Deferred) => {
9988                    // An unavailable fixer did not spend this repair request.
9989                    // Restore partial fallback work before retrying it, as for
9990                    // an interrupted repair, without closing the request.
9991                    let worktree = self
9992                        .state
9993                        .winner()
9994                        .context("repair has no winning branch")?
9995                        .worktree
9996                        .clone();
9997                    self.graft_restore(&worktree, &request.head).await?;
9998                    self.state.merge_repairs[i].phase = Phase::Requested;
9999                    self.stop_for_resource()?;
10000                    return Ok(false);
10001                }
10002                Ok(land::Fixed::Failed(why)) => {
10003                    self.hold_merge_repair(
10004                        i,
10005                        format!(
10006                            "repair failed: {why}; inspect the branch and run an explicit review"
10007                        ),
10008                    )?;
10009                    return Ok(false);
10010                }
10011                Err(e) => {
10012                    self.hold_merge_repair(
10013                        i,
10014                        format!(
10015                            "repair failed: {e:#}; inspect the branch and run an explicit review"
10016                        ),
10017                    )?;
10018                    return Ok(false);
10019                }
10020            }
10021            // Even no change gets a real new reviewer wave. The previous
10022            // evidence remains in the request; a former clean/concluded round
10023            // cannot short-circuit the new review or gate.
10024            self.begin_merge_repair_review(i)?;
10025        }
10026        self.review_loop().await?;
10027        self.gate().await?;
10028        if self.resource_halt()? || self.state.parked {
10029            return Ok(false);
10030        }
10031        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10032            != Some(RunStatus::Gating)
10033            || !self.state.gate_allows_progress()
10034        {
10035            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10036            return Ok(false);
10037        }
10038        // Review and gate may have added commits after the initial repair.
10039        // Publish exactly their final head to the existing PR, with a lease.
10040        let winner = self
10041            .state
10042            .winner()
10043            .cloned()
10044            .context("repair has no winning branch")?;
10045        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10046        if self.state.gate_head.as_deref() != Some(after.as_str())
10047            || !git::is_clean(&winner.worktree).await?
10048        {
10049            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10050            return Ok(false);
10051        }
10052        let expected = self
10053            .state
10054            .pr_pushed_head
10055            .as_deref()
10056            .unwrap_or(&request.head)
10057            .to_owned();
10058        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10059        // Push the checked commit itself: a local ref moving after the check
10060        // must not change which commit the forge receives.
10061        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10062            return Ok(false);
10063        }
10064        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10065        let refspec = format!("{after}:refs/heads/{public}");
10066        let push = git::git_raw(
10067            &winner.worktree,
10068            &["push", &lease, &self.state.config.merge.remote, &refspec],
10069        )
10070        .await;
10071        match push {
10072            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10073            Ok(out) => {
10074                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10075                return Ok(false);
10076            }
10077            Err(e) => {
10078                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10079                return Ok(false);
10080            }
10081        }
10082        self.state.merge_repairs[i].phase = Phase::Complete;
10083        self.state.status = RunStatus::Landing;
10084        self.state.event(
10085            "repair",
10086            "new review and gate completed; fresh owner merge decision required",
10087        );
10088        self.state.save()?;
10089        Ok(true)
10090    }
10091
10092    fn merge_repair_artifacts(&self) -> PathBuf {
10093        let artifacts = agent::artifacts_dir(&self.state.dir());
10094        match self
10095            .state
10096            .merge_repairs
10097            .last()
10098            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10099        {
10100            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10101            None => artifacts,
10102        }
10103    }
10104
10105    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10106        if let Some(e2e) = self.state.head_e2e.take() {
10107            self.state
10108                .verification_history
10109                .push(crate::run::VerificationEvidence {
10110                    superseded_by: None,
10111                    archived_at: Some(Timestamp::now()),
10112                    fixes_spent: Some(self.state.gate_fixes.len()),
10113                    gate_head: None,
10114                    gate_ran: false,
10115                    gate: Vec::new(),
10116                    e2e: Some(e2e),
10117                });
10118        }
10119        self.state.reviews.clear();
10120        self.state.gate.clear();
10121        self.state.gate_ran = false;
10122        self.state.gate_head = None;
10123        self.state.gate_fixes.clear();
10124        self.state.gate_repair_e2e = None;
10125        self.state.pre_gate.clear();
10126        self.state.pre_gate_commit = None;
10127        self.state.contested_handoff = None;
10128        self.state.land_reruns = None;
10129        self.state.land_approval = None;
10130        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10131        self.state.status = RunStatus::Reviewing;
10132        self.state.save()
10133    }
10134
10135    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10136        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10137        self.state.merge_repairs[index].failure = Some(why.clone());
10138        self.state.status = RunStatus::Blocked;
10139        self.state.event("repair", why);
10140        self.state.save()
10141    }
10142
10143    // -------------------------------------------------------------- helpers
10144
10145    /// Fetch or create a seat, keeping its conversation across nodes.
10146    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10147        if let Some(existing) = self.state.seats.get(key)
10148            && existing.agent == agent
10149        {
10150            return existing.clone();
10151        }
10152        // A seat that changes agent mints its session id from the agent too,
10153        // like a handover: the old agent's uuid is already taken by the CLI.
10154        let fresh = if self.state.seats.contains_key(key) {
10155            handover_seat(key, agent, self.state.next_seat_seed())
10156        } else {
10157            SeatState::new(key, agent, self.state.seed)
10158        };
10159        self.state.seats.insert(key.to_owned(), fresh.clone());
10160        fresh
10161    }
10162
10163    /// The agent now holding seat `key`: `spec`, unless a handover moved the
10164    /// seat to another roster agent, in which case that agent. Nodes that
10165    /// continue a seat's conversation (deliberation, the votes, a reviewer's
10166    /// reconsideration) must keep talking to whoever answered it, not slip
10167    /// back to the agent that failed it.
10168    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10169        match self.state.seats.get(key) {
10170            Some(s) if s.agent != spec.id => {
10171                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10172            }
10173            _ => spec,
10174        }
10175    }
10176
10177    /// A candidate rendered for judging, with the leak policy applied.
10178    fn view(&self, c: &Candidate) -> CandidateView {
10179        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10180            .unwrap_or_default();
10181        let (patch, _) = blind::sanitize_patch(
10182            &format!("candidate {} patch", c.label),
10183            &raw,
10184            &self.state.config.blind,
10185        );
10186        CandidateView {
10187            label: c.label,
10188            branch: c.branch.clone(),
10189            summary: c.summary.clone(),
10190            stat: c.stat.clone(),
10191            patch,
10192        }
10193    }
10194
10195    /// The full candidate set as prompt text, for seats with no live session.
10196    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10197        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10198        prompt::judge(
10199            "(see above)",
10200            &views,
10201            self.roles.judges.len(),
10202            base_short,
10203            "en",
10204        )
10205    }
10206
10207    /// The final-vote prompt with everything a seat that has no session of its
10208    /// own needs: the candidates, the seat's own ranking and reasons, and the
10209    /// anonymised deliberation it took part in (only when there was one, so a
10210    /// handed-over seat never sees more than the seat it replaces did). The
10211    /// `Final vote` heading stays first.
10212    fn vote_prompt_full(
10213        &self,
10214        j: usize,
10215        labels: &[char],
10216        language: &str,
10217        candidates: &[Candidate],
10218        base_short: &str,
10219    ) -> String {
10220        let mut text = format!(
10221            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10222            prompt::final_vote(labels, language),
10223            self.state.instruction,
10224            self.candidate_block(candidates, base_short)
10225        );
10226        if let Some(own) = self
10227            .state
10228            .judgements
10229            .get(j)
10230            .filter(|r| !r.ranking.is_empty())
10231        {
10232            let reasons = own
10233                .reasons
10234                .iter()
10235                .map(|(k, v)| format!("- {k}: {v}"))
10236                .collect::<Vec<_>>()
10237                .join("\n");
10238            text.push_str(&format!(
10239                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10240                own.ranking.iter().collect::<String>(),
10241                if reasons.is_empty() {
10242                    ""
10243                } else {
10244                    ", because:\n"
10245                }
10246            ));
10247        }
10248        if !self.state.deliberation.is_empty() {
10249            text.push_str("\n# What was argued before this vote\n");
10250            for t in self.transcript(&[], j) {
10251                text.push_str(&format!(
10252                    "\n## {}{}\n\n{}\n",
10253                    t.who,
10254                    if t.is_self { " (you)" } else { "" },
10255                    t.body.trim()
10256                ));
10257            }
10258        }
10259        text
10260    }
10261
10262    /// Anonymised transcript for judge `self_idx`.
10263    ///
10264    /// The initial rankings are always the opening statements. Seeding them
10265    /// only when no turn had been taken yet meant every judge after the first
10266    /// argued against a single voice instead of against the actual split — the
10267    /// disagreement is the information, so it is always on the table.
10268    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10269        let mut turns = Vec::new();
10270        for j in &self.state.judgements {
10271            if j.ranking.is_empty() {
10272                continue;
10273            }
10274            let reasons = j
10275                .reasons
10276                .iter()
10277                .map(|(k, v)| format!("- {k}: {v}"))
10278                .collect::<Vec<_>>()
10279                .join("\n");
10280            turns.push(Turn {
10281                who: format!("Judge {} (opening ranking)", j.judge),
10282                is_self: j.judge == self_idx + 1,
10283                body: format!(
10284                    "Ranked {}{}{reasons}",
10285                    j.ranking.iter().collect::<String>(),
10286                    if reasons.is_empty() {
10287                        ""
10288                    } else {
10289                        ", because:\n"
10290                    }
10291                ),
10292            });
10293        }
10294        for t in self
10295            .state
10296            .deliberation
10297            .iter()
10298            .flat_map(|r| r.turns.iter())
10299            .chain(current)
10300        {
10301            turns.push(Turn {
10302                who: format!("Judge {}", t.judge),
10303                is_self: t.judge == self_idx + 1,
10304                body: t.body.clone(),
10305            });
10306        }
10307        turns
10308    }
10309}
10310
10311/// Does this seat still hold the context a follow-up prompt would rely on?
10312fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10313    agent::has_session(spec.kind, seat, sessions)
10314}
10315
10316/// The next entry in `roster` after `start`, never wrapping back to the
10317/// front, whose id is not in `tried` yet.
10318///
10319/// Starts one past `start` rather than at the front of `roster`: `start` is
10320/// the seat's own original position, and a seat whose candidate slot already
10321/// sits on the roster's second entry must fall through to the third next, not
10322/// restart at the first — which is very likely a different candidate's own
10323/// agent already. Never wraps back past `start`, for the same reason: an
10324/// entry earlier in the roster than the seat's own position is almost
10325/// certainly some *other* candidate slot's own agent, and once the tail of
10326/// the roster is exhausted there are no more untried agents for *this* seat
10327/// to fall through to — the caller's fallback chain ends there, exactly as
10328/// "no further untried agents remain in the list for that seat" asks for.
10329///
10330/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
10331/// the same id twice (an operator's `roles.implementers` typo, or a
10332/// `[[agents]]` list reused across roles) must not let
10333/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
10334/// over `roster` either finds an untried id or runs out, so this always
10335/// terminates regardless of duplicates.
10336fn next_untried_in_roster<'a>(
10337    roster: &'a [AgentSpec],
10338    start: usize,
10339    tried: &BTreeSet<String>,
10340) -> Option<&'a AgentSpec> {
10341    roster
10342        .get(start + 1..)?
10343        .iter()
10344        .find(|s| !tried.contains(&s.id))
10345}
10346
10347/// The successor for a seat, shared by all four seat kinds (implement, judge,
10348/// review, advise). `others` holds the ids that *currently* occupy the other
10349/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
10350/// sees them), so roster entries beyond the seat count act as spares: a failed
10351/// seat goes to an agent no other seat holds whenever the roster permits.
10352///
10353/// `carried` is `Some` only for the review loop's carried failure history.
10354/// Order: (1) forward from `start`, never wrapping, untried, not a carried
10355/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
10356/// whole roster: untried and not another seat's occupant; (3) the plain walk
10357/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
10358/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
10359/// a better outcome than an empty seat, and it keeps the answer to "is there
10360/// a successor at all" exactly what it was before occupants were considered,
10361/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
10362/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
10363/// carried failure is a cheaper bet than doubling an agent on the panel, and
10364/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
10365/// the same round are handled one at a time, so a seat later in the batch
10366/// sees an earlier one's new occupant but not yet its own freed agent.
10367fn pick_successor<'a>(
10368    roster: &'a [AgentSpec],
10369    start: usize,
10370    tried: &BTreeSet<String>,
10371    carried: Option<&BTreeSet<String>>,
10372    others: &BTreeSet<String>,
10373) -> Option<&'a AgentSpec> {
10374    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10375    roster
10376        .get(start + 1..)
10377        .and_then(|tail| {
10378            tail.iter()
10379                .filter(free)
10380                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10381        })
10382        .or_else(|| {
10383            carried
10384                .is_some()
10385                .then(|| roster.iter().find(free))
10386                .flatten()
10387        })
10388        .or_else(|| match carried {
10389            Some(c) => next_for_seat(roster, start, tried, c),
10390            None => next_untried_in_roster(roster, start, tried),
10391        })
10392}
10393
10394/// The next agent for a seat that carries its failure history across rounds
10395/// (the review loop). `round_tried` is this round's own bound and starts
10396/// empty every round; `carried_failed` only decides priority.
10397///
10398/// First: an id walking forward from `start`, never wrapping, that is neither
10399/// tried this round nor failed in an earlier one. Only when that is exhausted
10400/// does it rescue: the first roster id (in roster order, so this one *does*
10401/// look before `start`) not yet tried this round, which is by then a carried
10402/// failure. Each id is rescued at most once per round, so it cannot loop.
10403fn next_for_seat<'a>(
10404    roster: &'a [AgentSpec],
10405    start: usize,
10406    round_tried: &BTreeSet<String>,
10407    carried_failed: &BTreeSet<String>,
10408) -> Option<&'a AgentSpec> {
10409    roster
10410        .get(start + 1..)?
10411        .iter()
10412        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10413        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10414}
10415
10416/// Where a reviewer seat starts a round: the agent that last answered it when
10417/// it is still on the roster and not marked failed, else the spec's own agent
10418/// unless it failed, else the next roster agent that has not failed, else the
10419/// spec's own agent again (the whole roster failed). Ids no longer on the
10420/// roster are ignored. An empty roster has no handover, so the spec stands.
10421fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10422    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10423        return spec;
10424    };
10425    let ok = |id: &str| !h.failed.contains(id);
10426    if let Some(last) = h.last_ok.as_deref()
10427        && ok(last)
10428        && let Some(s) = roster.iter().find(|s| s.id == last)
10429    {
10430        return s.clone();
10431    }
10432    if ok(&spec.id) {
10433        return spec;
10434    }
10435    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10436    roster
10437        .get(from + 1..)
10438        .into_iter()
10439        .flatten()
10440        .chain(roster.iter())
10441        .find(|s| ok(&s.id))
10442        .cloned()
10443        .unwrap_or(spec)
10444}
10445
10446/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
10447/// seed so a CLI that mints its session id up front (`--session-id`) never
10448/// reuses the uuid the previous agent already opened under the same seat key.
10449pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10450    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10451}
10452
10453/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
10454/// so the classifier below and the code that builds the message cannot drift.
10455const TIMED_OUT: &str = "timed out";
10456
10457impl FailClass {
10458    /// `None` for an answer; otherwise how the turn failed.
10459    fn of(out: &AgentOutcome) -> Option<Self> {
10460        match out {
10461            AgentOutcome::Ok(_) => None,
10462            AgentOutcome::Quota(_) => Some(Self::Quota),
10463            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10464            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10465            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10466            // Not a failure of the agent. Callers check for it first; `None`
10467            // here means "no class to hand over on".
10468            AgentOutcome::ResourceBlocked(_) => None,
10469        }
10470    }
10471
10472    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
10473    fn stem_word(&self) -> &'static str {
10474        match self {
10475            Self::Quota => "quota",
10476            _ => "handover",
10477        }
10478    }
10479}
10480
10481/// The message's first line with its variable parts removed — digit runs and
10482/// path-like tokens — so "exit code 2" and "exit code 7" read
10483/// as one kind of failure.
10484fn failure_signature(msg: &str) -> String {
10485    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10486    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
10487    // the historical machine key while the process diagnosis uses readable prose.
10488    if let Some(code) = line
10489        .strip_prefix("process exited with no usable output (exit code ")
10490        .and_then(|s| s.strip_suffix(')'))
10491        .and_then(|s| s.parse::<i32>().ok())
10492    {
10493        line = format!("exited with some({code}) and no usable output");
10494    } else if matches!(
10495        line.as_str(),
10496        "process exited with no usable output (no exit code reported)"
10497            | "process exited with no usable output (exit code unavailable)"
10498    ) {
10499        line = "exited with none and no usable output".to_owned();
10500    }
10501    let mut out = Vec::new();
10502    for word in line.split_whitespace() {
10503        if word.contains('/') || word.contains('\\') {
10504            out.push("<path>".to_owned());
10505            continue;
10506        }
10507        let mut w = String::new();
10508        let mut in_digits = false;
10509        for c in word.chars() {
10510            if c.is_ascii_digit() {
10511                if !in_digits {
10512                    w.push('#');
10513                }
10514                in_digits = true;
10515            } else {
10516                in_digits = false;
10517                w.push(c);
10518            }
10519        }
10520        out.push(w);
10521    }
10522    out.join(" ").chars().take(120).collect()
10523}
10524
10525/// Whether a seat that just failed with `cur` may go to the next roster agent.
10526/// A quota or a timeout always may. Any other failure may not when the agent
10527/// before it failed the same way: an error the prompt causes would otherwise
10528/// walk the whole roster. `prev` is the class of the immediately preceding
10529/// agent's failure, so a quota or timeout in between breaks the run of
10530/// identical failures by itself.
10531fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10532    match cur {
10533        FailClass::Quota | FailClass::Timeout => true,
10534        FailClass::Other(_) => prev != Some(cur),
10535    }
10536}
10537
10538/// A short human reason for a failed outcome, for the handover record.
10539fn fail_reason(out: &AgentOutcome) -> String {
10540    match out {
10541        AgentOutcome::Ok(_) => String::new(),
10542        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10543        AgentOutcome::Dropped(o) => format!(
10544            "the CLI dropped the stream ({})",
10545            o.dropped
10546                .as_ref()
10547                .map(|d| d.why.as_str())
10548                .unwrap_or("it ended without delivering its answer")
10549        ),
10550        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10551            e.lines().next().unwrap_or("").chars().take(160).collect()
10552        }
10553    }
10554}
10555
10556/// Note one handover in the run: the structured record and, in the timeline,
10557/// the sentence a person reads. A quota keeps the wording it always had.
10558pub(crate) fn record_handover(
10559    state: &mut RunState,
10560    node: &str,
10561    seat: &str,
10562    from: &str,
10563    to: &str,
10564    class: &FailClass,
10565    reason: &str,
10566) {
10567    let message = if *class == FailClass::Quota {
10568        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10569    } else {
10570        format!("{seat}: handed over {from} -> {to} ({reason})")
10571    };
10572    state.event(node, message);
10573    state.handovers.push(Handover {
10574        at: Timestamp::now(),
10575        node: node.to_owned(),
10576        seat: seat.to_owned(),
10577        from: from.to_owned(),
10578        to: to.to_owned(),
10579        reason: reason.to_owned(),
10580    });
10581}
10582
10583/// Did this reply report running a command whose own CLI never confirmed an
10584/// exit status?
10585///
10586/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
10587/// command *finished* (see that type's own doc), so this can only be `true`
10588/// for a command whose completion event carried no readable exit code — not
10589/// for one that simply is not mentioned at all. That is the one signal this
10590/// crate can read, from the same record `state.jobs` renders, about a reply
10591/// standing next to work its own CLI cannot vouch for finishing; it is
10592/// deliberately not a check on the exit code's *value* (a fixer legitimately
10593/// runs a command that fails mid-iteration before it succeeds) and not a
10594/// guess at a command still running in the background (which emits no event
10595/// at all, and so leaves no evidence here to find).
10596fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10597    commands.iter().any(|c| c.exit_code.is_none())
10598}
10599
10600/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
10601/// trusted as a verified no-op — the adoption guard's own text-level half.
10602///
10603/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
10604/// timed out): a marker only earns the benefit of the doubt from a turn the
10605/// CLI itself vouches for finishing properly, the same house style
10606/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
10607/// *fix* report to for `commands`. A candidate that timed out, exited
10608/// non-zero, or left a command unconfirmed is read as the ordinary loss it
10609/// is, whatever prose it wrote — this returns `None` before it ever looks at
10610/// `text`. The remaining guards (the tree really is empty, the evidence is
10611/// non-empty) are the caller's: this only reads what the reply *claimed*.
10612fn verified_noop_claim(
10613    usable: bool,
10614    commands: &[agent::CommandEvidence],
10615    text: &str,
10616) -> Option<String> {
10617    (usable && !has_unconfirmed_command(commands))
10618        .then(|| verdict::verified_noop(text))
10619        .flatten()
10620}
10621
10622fn short(commit: &str) -> String {
10623    commit.chars().take(7).collect()
10624}
10625
10626fn make_executable(path: &Path) -> Result<()> {
10627    #[cfg(unix)]
10628    {
10629        use std::os::unix::fs::PermissionsExt as _;
10630        let mut perms = std::fs::metadata(path)?.permissions();
10631        perms.set_mode(0o755);
10632        std::fs::set_permissions(path, perms)?;
10633    }
10634    #[cfg(not(unix))]
10635    {
10636        let _ = path;
10637    }
10638    Ok(())
10639}
10640
10641/// What every seat in one batch shares: where the answers are attributed, the
10642/// prompt overlay they inherit, and the build cache they are told to use.
10643///
10644/// A struct rather than four more parameters: `wave` also needs the run's
10645/// state (to record who is answering right now) and the attempt number, and
10646/// eight positional arguments is both unreadable and a clippy error.
10647struct WaveCtx<'a> {
10648    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
10649    /// paid for it.
10650    run: &'a str,
10651    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
10652    node: &'a str,
10653    prompts: &'a Prompts,
10654    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
10655    cache: Option<&'a Path>,
10656    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
10657    /// `JobRecord::round`. `None` for every other node.
10658    round: Option<usize>,
10659    /// Carry each seat's failed-agent history across waves (the review loop
10660    /// only): start-of-round priority and handover choice read
10661    /// [`RunState::seat_history`], and every answer or failure writes it.
10662    carry_seats: bool,
10663    /// The run's park / interrupt requests, so a wait for a shared resource
10664    /// can be abandoned the moment the operator asks to park.
10665    pauses: Option<(&'a Pause, &'a Pause)>,
10666}
10667
10668/// Shared scheduling policy for operations and spawned seat invocations.
10669fn repository_wait(
10670    state: &RunState,
10671    pauses: Option<(&Pause, &Pause)>,
10672) -> crate::repo_lock::WaitPolicy {
10673    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10674    let inherited = crate::repo_lock::WaitPolicy::current();
10675    crate::repo_lock::WaitPolicy::new(
10676        Duration::from_secs(state.config.graph.timeout_resource_wait),
10677        move || {
10678            inherited.cancelled()
10679                || pauses
10680                    .as_ref()
10681                    .is_some_and(|(a, b)| a.parked() || b.parked())
10682        },
10683    )
10684}
10685
10686fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10687    if error.is::<crate::repo_lock::ResourceBlocked>() {
10688        state.resource_block = Some(crate::run::ResourceBlock {
10689            cause: crate::run::ResourceBlockCause::Other,
10690            node: "repository".into(),
10691            kind: if error.to_string().contains("disk pressure") {
10692                "disk pressure"
10693            } else if error.to_string().contains("disk admission") {
10694                "disk admission"
10695            } else if error.to_string().contains(crate::incremental::BLOCKED) {
10696                "incremental cache admission"
10697            } else {
10698                "repository coordination"
10699            }
10700            .into(),
10701            detail: error.to_string(),
10702            at: Timestamp::now(),
10703        });
10704        state.event("resource", error.to_string());
10705    }
10706}
10707
10708async fn coordinate_operation(
10709    state: &mut RunState,
10710    cwd: &Path,
10711    wait: &crate::repo_lock::WaitPolicy,
10712) -> Result<Option<crate::repo_lock::SeatLease>> {
10713    let result =
10714        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10715            .await;
10716    if let Err(e) = &result {
10717        record_repository_block(state, e);
10718    }
10719    result
10720}
10721
10722/// Run one job, honouring the parallelism budget.
10723async fn run_one(
10724    job: SeatJob,
10725    sem: Arc<Semaphore>,
10726    ctx: &WaveCtx<'_>,
10727    state: &mut RunState,
10728    attempt: usize,
10729) -> (SeatState, AgentOutcome) {
10730    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10731        .await
10732        .pop()
10733        .expect("one job in, one result out");
10734    (seat, out)
10735}
10736
10737/// Run every job concurrently, capped by the semaphore, preserving order.
10738///
10739/// Every seat in the batch is recorded into [`RunState::active`] before the
10740/// wave starts and cleared as each answer lands, so the run's own record says
10741/// who is still being waited on rather than only who finished.
10742async fn wave(
10743    jobs: Vec<SeatJob>,
10744    sem: Arc<Semaphore>,
10745    ctx: &WaveCtx<'_>,
10746    state: &mut RunState,
10747    attempt: usize,
10748) -> Vec<(usize, SeatState, AgentOutcome)> {
10749    let WaveCtx {
10750        run,
10751        node,
10752        prompts,
10753        cache,
10754        round,
10755        carry_seats: _,
10756        pauses,
10757    } = *ctx;
10758    for job in &jobs {
10759        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10760    }
10761    if let Err(e) = state.save() {
10762        // A failed persist of "who is answering right now" must not abort the
10763        // wave: the seats are already being asked, and the alternative is
10764        // losing the answers to save a status line nobody may even be
10765        // watching.
10766        tracing::warn!("could not persist in-progress seats: {e:#}");
10767    }
10768    // Hold the shared build cache's lease for the whole batch, not per job:
10769    // several candidates (an implement wave) or a fixer legitimately share
10770    // one cache concurrently within this run, and that stays untouched — a
10771    // single lease taken once for the whole wave and released once it is
10772    // done is what stops a *different* borrower (another run's own wave, its
10773    // e2e/gate, a human's `magi review`) from interleaving a build into the
10774    // same directory while this one is in flight. Best-effort, not
10775    // all-or-nothing: a wave that cannot get the lease within its own
10776    // longest job's budget still runs — an hour of paid implementer calls is
10777    // not thrown away over cache contention — but every write-allowed seat
10778    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
10779    // below), the same fallback a read-only seat always gets, rather than
10780    // building into a directory this run was never granted. The identity
10781    // record is still invalidated below either way, so the next tracked
10782    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
10783    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10784    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10785    // never borrowed from a job's agent budget (and never charged to it either,
10786    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10787    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10788    let cache_guard = if let Some(cache_dir) = cache {
10789        if jobs_had_a_writer {
10790            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10791            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10792            match acquire_cache_lease(
10793                state,
10794                cache_dir,
10795                &owner,
10796                resource_limit,
10797                node,
10798                node,
10799                &cancel,
10800            )
10801            .await
10802            {
10803                Ok(Some(g)) => Some(g),
10804                // Parking: leave without spawning anything. The run parks at
10805                // the next boundary and the seats are asked afresh on resume.
10806                Ok(None) => {
10807                    if let Some(block) = &mut state.resource_block {
10808                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10809                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10810                        };
10811                    }
10812                    return block_wave(
10813                        jobs,
10814                        state,
10815                        node,
10816                        attempt,
10817                        "parked while waiting for the shared build cache",
10818                    );
10819                }
10820                // An unreadable lease is not contention: keep the old
10821                // fallback (no shared cache for this wave's seats).
10822                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10823                Err(e) => {
10824                    // Writing seats may not build into a cache this run does
10825                    // not hold, and an agent that cannot be given one is not
10826                    // asked at all: the run stops resumably instead.
10827                    let why = format!("{e:#}");
10828                    state.event(
10829                        "resource",
10830                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10831                    );
10832                    return block_wave(jobs, state, node, attempt, &why);
10833                }
10834            }
10835        } else {
10836            None
10837        }
10838    } else {
10839        None
10840    };
10841    // Every job keeps its whole budget: the wait above was not the agent's.
10842    // Restart the seats' clocks so "started" means "being spawned now".
10843    state.seats_spawning(node, attempt);
10844    if let Err(e) = state.save() {
10845        tracing::warn!("could not persist the start of a wave: {e:#}");
10846    }
10847    // Queueing behind this wave's own seats for a parallelism slot is normal
10848    // (each of them is bounded by its own agent timeout); only a wait beyond
10849    // all of them plus the resource limit means the slots never came back.
10850    let slot_limit = resource_limit
10851        + jobs
10852            .iter()
10853            .map(|j| j.timeout)
10854            .max()
10855            .unwrap_or_default()
10856            .saturating_mul(jobs.len() as u32);
10857    let mut set = tokio::task::JoinSet::new();
10858    // Seats announce themselves here the moment they hold a parallelism
10859    // permit, i.e. just before their CLI is spawned.
10860    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10861    let overlay = prompts.overlay(node);
10862    let repository_wait = repository_wait(state, pauses);
10863    for (i, mut job) in jobs.into_iter().enumerate() {
10864        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10865        if cache.is_some() {
10866            job.prompt.push('\n');
10867            job.prompt
10868                .push_str(&prompt::build_cache_note(node, job.allow_write));
10869        }
10870        let sem = Arc::clone(&sem);
10871        let run = run.to_owned();
10872        let node = node.to_owned();
10873        // Only implementers were told about the task's attachments, so only
10874        // their seats get the directory widened for reading.
10875        let attachments = if node == "implement" {
10876            state.attachments.clone()
10877        } else {
10878            Vec::new()
10879        };
10880        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10881        // `prompt::build_cache_note`'s doc for why setting it anyway is
10882        // exactly how a sandboxed reviewer's write refusal got reported as a
10883        // defect in the patch, not a property of its own seat. And a
10884        // write-allowed one is handed it only when the lease above was
10885        // actually acquired: a wave that could not get it (`cache_guard` is
10886        // `None`, see its own comment) must not send seats to build into a
10887        // directory this run does not hold - that is the exact concurrent,
10888        // unmanaged-write race this module exists to prevent, not something
10889        // "proceeding anyway" is allowed to reintroduce.
10890        let cache = cache
10891            .filter(|_| job.allow_write && cache_guard.is_some())
10892            .map(Path::to_path_buf);
10893        let spawned_tx = spawned_tx.clone();
10894        let repository_wait = repository_wait.clone();
10895        let coordination = state.config.coordination.clone();
10896        let fetched_base = state.fetched_base.clone();
10897        set.spawn(async move {
10898            // A wait for a parallelism slot is scheduling, not the agent: it
10899            // is bounded by its own limit and ends as a resource block, never
10900            // as a timeout of the agent that was not started.
10901            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10902                Ok(p) => p,
10903                Err(_) => {
10904                    return (
10905                        i,
10906                        job.seat,
10907                        AgentOutcome::ResourceBlocked(format!(
10908                            "no parallel slot became free within {}s",
10909                            slot_limit.as_secs()
10910                        )),
10911                        false,
10912                    );
10913                }
10914            };
10915            spawned_tx.send(job.seat.key.clone()).ok();
10916            let mut seat = job.seat;
10917            let out = repository_wait
10918                .scope(agent::invoke(
10919                    &job.spec,
10920                    &mut seat,
10921                    &Invocation {
10922                        coordination: coordination.as_ref(),
10923                        fetched_base: fetched_base.as_ref(),
10924                        cwd: &job.cwd,
10925                        prompt: &job.prompt,
10926                        timeout: job.timeout,
10927                        allow_write: job.allow_write,
10928                        sessions: job.sessions,
10929                        artifacts: &job.artifacts,
10930                        stem: &job.stem,
10931                        run: &run,
10932                        node: &node,
10933                        cache_dir: cache.as_deref(),
10934                        attachments: &attachments,
10935                        writable: &[],
10936                    },
10937                ))
10938                .await;
10939            let interrupted = out
10940                .as_ref()
10941                .err()
10942                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10943            let out = match out {
10944                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10945                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10946                // Billed work the CLI failed to hand over is not an ordinary
10947                // failure, but its text is the CLI's raw error JSON, not an
10948                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
10949                // read it as one by forgetting to check. `usable()` is always
10950                // false here (dropped implies an empty response), so this has
10951                // to be checked before the catch-all `Failed` below or the
10952                // one shape this exists for is lost with the rest.
10953                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10954                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10955                Ok(o) => AgentOutcome::Failed(format!(
10956                    "process exited with no usable output ({})",
10957                    crate::notices::exit_status(o.exit_code)
10958                )),
10959                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10960                    AgentOutcome::ResourceBlocked(e.to_string())
10961                }
10962                Err(e) => AgentOutcome::Failed(e.to_string()),
10963            };
10964            (i, seat, out, interrupted)
10965        });
10966    }
10967    drop(spawned_tx);
10968    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10969    loop {
10970        let joined = tokio::select! {
10971            Some(key) = spawned_rx.recv() => {
10972                // The clock of this seat starts now, not when it queued.
10973                state.seat_spawned(&key);
10974                if let Err(e) = state.save() {
10975                    tracing::warn!("could not persist a seat's start: {e:#}");
10976                }
10977                continue;
10978            }
10979            joined = set.join_next() => joined,
10980        };
10981        let Some(joined) = joined else {
10982            break;
10983        };
10984        let (i, seat, out, interrupted) = match joined {
10985            Ok(v) => v,
10986            // No seat to clear: a panicked task never reported which one it
10987            // was. The defensive sweep below this loop is what stops that
10988            // seat's `active` entry from surviving forever.
10989            Err(e) => {
10990                tracing::error!("agent task panicked: {e}");
10991                continue;
10992            }
10993        };
10994        state.seat_finished(&seat.key);
10995        if interrupted
10996            && node == "implement"
10997            && let Some(label) = state
10998                .candidates
10999                .iter()
11000                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
11001                .map(|candidate| candidate.label)
11002        {
11003            state.candidate_preservation.insert(
11004                label,
11005                "interrupted implementation awaiting persistence".into(),
11006            );
11007            state.interrupted_implementations.insert(label);
11008        }
11009        if let AgentOutcome::ResourceBlocked(why) = &out
11010            && state.resource_block.is_none()
11011        {
11012            state.resource_block = Some(crate::run::ResourceBlock {
11013                cause: crate::run::ResourceBlockCause::Other,
11014                node: node.to_owned(),
11015                kind: if why.contains("disk pressure") {
11016                    "disk pressure"
11017                } else if why.contains("disk admission") {
11018                    "disk admission"
11019                } else if why.contains(crate::incremental::BLOCKED) {
11020                    "incremental cache admission"
11021                } else if why.contains("repository coordination") {
11022                    "repository coordination"
11023                } else {
11024                    "parallel slot"
11025                }
11026                .to_owned(),
11027                detail: why.clone(),
11028                at: Timestamp::now(),
11029            });
11030            let activity = if interrupted {
11031                "agent execution interrupted"
11032            } else {
11033                "no agent was started"
11034            };
11035            state.event("resource", format!("{node}: {activity} - {why}"));
11036        }
11037        record_jobs(state, node, round, &seat.key, &out);
11038        if let Err(e) = state.save() {
11039            tracing::warn!("could not persist a seat's completion: {e:#}");
11040        }
11041        if collected.len() <= i {
11042            collected.resize_with(i + 1, || None);
11043        }
11044        collected[i] = Some((i, seat, out));
11045    }
11046    // Belt-and-braces for the panic branch above: every seat this exact batch
11047    // started shares this `(node, attempt)` pair, and every seat that finished
11048    // normally already cleared itself, so anything left tagged with it here
11049    // can only be a panicked task's leftover. Cleared unconditionally rather
11050    // than left to read as still answering forever.
11051    if state
11052        .active
11053        .values()
11054        .any(|a| a.node == node && a.attempt == attempt)
11055    {
11056        state
11057            .active
11058            .retain(|_, a| !(a.node == node && a.attempt == attempt));
11059        if let Err(e) = state.save() {
11060            tracing::warn!("could not persist the end of a wave: {e:#}");
11061        }
11062    }
11063    // Whether or not the lease above was actually held, several worktrees
11064    // may just have built into the cache with nothing here able to name one
11065    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
11066    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
11067    // clean it might not have strictly needed; trusting a stale match would
11068    // cost it a wrong answer.
11069    if let Some(cache_dir) = cache
11070        && jobs_had_a_writer
11071    {
11072        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11073    }
11074    if let Some(guard) = cache_guard {
11075        guard.release();
11076    }
11077    collected.into_iter().flatten().collect()
11078}
11079
11080/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
11081/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
11082/// this is data collection, not the fix-specific completion contract in
11083/// [`Runner::continue_fix_report`], and applies regardless of which node
11084/// asked.
11085///
11086/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
11087/// evidence from; `Failed` does not, and correctly contributes nothing — a
11088/// timeout or crash is not itself evidence about a command the seat may have
11089/// started.
11090fn record_jobs(
11091    state: &mut RunState,
11092    node: &str,
11093    round: Option<usize>,
11094    seat: &str,
11095    out: &AgentOutcome,
11096) {
11097    let commands: &[agent::CommandEvidence] = match out {
11098        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11099        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11100    };
11101    let checked_at = Timestamp::now();
11102    for c in commands {
11103        state.jobs.push(JobRecord {
11104            node: node.to_owned(),
11105            round,
11106            seat: seat.to_owned(),
11107            id: c.id.clone(),
11108            description: c.description.clone(),
11109            checked_at,
11110            status: match c.exit_code {
11111                Some(0) => JobStatus::Completed,
11112                Some(_) => JobStatus::Failed,
11113                None => JobStatus::Unknown,
11114            },
11115            exit_code: c.exit_code,
11116            result_summary: c.result_summary.clone(),
11117            source: c.source.clone(),
11118        });
11119    }
11120}
11121
11122/// Is a review round clean, given how many reviewer seats answered against
11123/// how many the round expected?
11124///
11125/// A seat that never answered (timeout, crash, unparsable output) is not a
11126/// seat that read the patch and found nothing — treating it as such is
11127/// exactly the bug this function exists to close. Under the default `block`
11128/// policy a missing seat can never be clean; `warn` still requires the seats
11129/// that *did* answer to have found nothing blocking and verification to be
11130/// green.
11131///
11132/// `quota_missing` narrows that `block` default for exactly one cause of
11133/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
11134/// a session limit lifts by the very next round buys nothing — the seat is
11135/// asked again with the same quota — so once every missing seat is accounted
11136/// for by a quota loss (and at least one seat *did* answer, so a decision has
11137/// something to rest on) the round is decided on the panel that could answer,
11138/// same as `warn` would. A panel that lost every seat to quota is not
11139/// decided here: `answered == 0` falls through to the existing `block`
11140/// fallback so a fully collapsed panel still waits rather than landing on no
11141/// review at all.
11142fn round_is_clean(
11143    blocking: usize,
11144    e2e_ok: bool,
11145    answered: usize,
11146    expected: usize,
11147    quota_missing: usize,
11148    policy: IncompleteReviewPolicy,
11149) -> bool {
11150    if blocking != 0 || !e2e_ok {
11151        return false;
11152    }
11153    if answered == expected || policy == IncompleteReviewPolicy::Warn {
11154        return true;
11155    }
11156    answered > 0 && expected - answered <= quota_missing
11157}
11158
11159/// The review loop's own conclusion, derived entirely from its persisted
11160/// round records and the round budget that produced them — never from
11161/// `status`, so a reentry (or `gate`/`merge` reading it independently)
11162/// recomputes the identical answer regardless of what an earlier node in the
11163/// same walk, or a previous walk, did to `status`.
11164///
11165/// `None` while more rounds remain to try, including when review never ran
11166/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
11167/// gone clean, or the budget is spent, or the tree has stopped moving (see
11168/// [`STAGNANT_LIMIT`]), the answer is one of two things:
11169///
11170/// - An incomplete panel that raised nothing is missing input, not a
11171///   verified tree — never a hand-off candidate, whatever verification said
11172///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
11173/// - Otherwise, green e2e on the last round hands off (see
11174///   [`Runner::stop_reviewing`]); red e2e blocks.
11175///
11176/// A last round whose own verification is still `ResourceBlocked` — magi
11177/// itself never got a command to run, not evidence the patch is broken —
11178/// is neither: this returns `None` for it too, the same as "more rounds
11179/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
11180/// handling of that shape) instead of this cheap recomputation guessing a
11181/// verdict a real attempt never produced.
11182pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11183    if max_rounds == 0
11184        || reviews.last().is_some_and(|r| {
11185            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11186        })
11187    {
11188        return Some(RunStatus::Gating);
11189    }
11190    let last = reviews.last()?;
11191    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11192    if reviews.len() < max_rounds && !stagnant {
11193        return None;
11194    }
11195    if last.incomplete() && last.blocking == 0 {
11196        return Some(RunStatus::Blocked);
11197    }
11198    if last.e2e_status() == E2eStatus::ResourceBlocked
11199        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11200    {
11201        return None;
11202    }
11203    Some(if last.e2e.iter().all(CommandOutcome::ok) {
11204        RunStatus::Gating
11205    } else {
11206        RunStatus::Blocked
11207    })
11208}
11209
11210/// How long a re-ask may take, given the budget the first attempt had.
11211///
11212/// A `nudged` retry is a request to restate an answer the seat has already
11213/// worked out: it carries no new work, so it does not deserve the original
11214/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
11215/// 133 seconds while a third sat for over ten minutes on a resumed session
11216/// holding 410 KB of prior output - and because the retry had inherited the
11217/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
11218/// judging round whose other seats were long finished.
11219///
11220/// A quarter of the budget, with a floor so that a deliberately short timeout
11221/// does not collapse to nothing. A retry that re-sends the whole prompt
11222/// (because the seat kept no context) is the original job again, and keeps the
11223/// original budget.
11224pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11225    if nudged {
11226        (full / 4).max(Duration::from_secs(120)).min(full)
11227    } else {
11228        full
11229    }
11230}
11231
11232/// Run a wave and parse each reply, re-asking the seats whose reply was
11233/// unusable.
11234///
11235/// The re-ask is a nudge rather than the whole prompt again when the seat still
11236/// holds its conversation, which is the difference between a cheap retry and
11237/// paying for the entire candidate set twice.
11238///
11239/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
11240/// successor in `roster` is handed to it instead of being re-asked: the
11241/// handover is the retry. A seat with no successor left (a single-agent
11242/// roster, the roster's tail) is nudged as before, up to `retries` times. So
11243/// the asks to one seat in one node number at most
11244/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
11245/// is asked once (a dropped stream is nudged first), and only the last agent
11246/// of the chain gets the `retries` same-agent nudges. Each roster agent is
11247/// tried at most once per seat, walking forward from the seat's own position and never wrapping
11248/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
11249/// other failure stops the chain when the previous agent failed the same way
11250/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
11251/// [`has_context`] is false and the job's own full prompt and full budget are
11252/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
11253/// intermediate ones are not losses) and is returned as a failure like any
11254/// other absent seat — the caller decides whether the panel still has a
11255/// quorum. An empty `roster` disables handover: failures are nudged as they
11256/// always were, and a quota is simply lost. A reply that fails to parse or
11257/// validate is the prompt's doing and is only ever nudged, never handed over.
11258///
11259/// The returned [`SeatState`] names the agent that answered (or tried last).
11260#[allow(clippy::too_many_arguments)]
11261async fn ask_json_wave<T>(
11262    jobs: Vec<SeatJob>,
11263    sem: Arc<Semaphore>,
11264    retries: usize,
11265    roster: &[AgentSpec],
11266    ctx: &WaveCtx<'_>,
11267    losses: &mut Vec<QuotaLoss>,
11268    state: &mut RunState,
11269    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11270) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11271where
11272    T: serde::de::DeserializeOwned + Send + 'static,
11273{
11274    ask_wave_with(
11275        jobs,
11276        sem,
11277        retries,
11278        roster,
11279        ctx,
11280        losses,
11281        state,
11282        &|text: &str| {
11283            let v = verdict::extract_json::<T>(text)?;
11284            validate(&v)?;
11285            Ok(v)
11286        },
11287    )
11288    .await
11289}
11290
11291/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
11292/// a node whose answer is prose (deliberation) shares the same handover,
11293/// failure classification, quota bookkeeping and bounds instead of a copy.
11294///
11295/// A seat handed to another roster agent is sent the job's `handover` prompt
11296/// (when it has one) rather than `prompt`: the new agent has no session, so a
11297/// resume-style prompt would be incomplete. That holds for the handover ask
11298/// and for every nudge to that agent whose `has_context` is false.
11299#[allow(clippy::too_many_arguments)]
11300async fn ask_wave_with<T>(
11301    jobs: Vec<SeatJob>,
11302    sem: Arc<Semaphore>,
11303    retries: usize,
11304    roster: &[AgentSpec],
11305    ctx: &WaveCtx<'_>,
11306    losses: &mut Vec<QuotaLoss>,
11307    state: &mut RunState,
11308    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11309) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11310where
11311    T: Send + 'static,
11312{
11313    let n = jobs.len();
11314    let originals: Vec<SeatJob> = jobs;
11315    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11316    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11317    // Nudges each seat's *current* agent has taken — 0 for a first-ask
11318    // answer, N once it has gone through N nudges. Read back once this
11319    // returns, so a caller building a history record (`ReviewRecord`) can
11320    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
11321    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
11322    // field's own doc.
11323    let mut nudges: Vec<usize> = vec![0; n];
11324    // The agent now occupying each seat, the ids it has already been through,
11325    // where in the roster the walk began, the class of the last failure, and
11326    // the stem word of a handover not yet asked (full prompt, full budget).
11327    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11328    let mut tried: Vec<BTreeSet<String>> = specs
11329        .iter()
11330        .map(|s| BTreeSet::from([s.id.clone()]))
11331        .collect();
11332    let starts: Vec<usize> = specs
11333        .iter()
11334        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11335        .collect();
11336    let carry = ctx.carry_seats && !roster.is_empty();
11337    // Carried across rounds: ids that failed the seat earlier, and how the
11338    // last failure went (so a repeat of it is not handed over again).
11339    let carried: Vec<BTreeSet<String>> = originals
11340        .iter()
11341        .map(|j| {
11342            state
11343                .seat_history
11344                .get(&j.seat.key)
11345                .filter(|_| carry)
11346                .map(|h| h.failed.clone())
11347                .unwrap_or_default()
11348        })
11349        .collect();
11350    let mut prev: Vec<Option<FailClass>> = originals
11351        .iter()
11352        .map(|j| {
11353            state
11354                .seat_history
11355                .get(&j.seat.key)
11356                .filter(|_| carry)
11357                .and_then(|h| h.last_fail.clone())
11358        })
11359        .collect();
11360    let next_agent =
11361        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11362            let others: BTreeSet<String> = specs
11363                .iter()
11364                .enumerate()
11365                .filter(|(j, _)| *j != i)
11366                .map(|(_, s)| s.id.clone())
11367                .collect();
11368            pick_successor(
11369                roster,
11370                starts[i],
11371                tried,
11372                carry.then(|| &carried[i]),
11373                &others,
11374            )
11375            .cloned()
11376        };
11377    let mut fresh: Vec<Option<String>> = vec![None; n];
11378    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11379    let mut pending: Vec<usize> = (0..n).collect();
11380
11381    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
11382    // asks: an agent with a successor is asked once and handed over, and only
11383    // a seat with no successor spends `retries` nudges on the same agent. This
11384    // only guarantees the loop's own termination whatever those say.
11385    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11386    for round in 0..max_rounds {
11387        if pending.is_empty() {
11388            break;
11389        }
11390        let mut batch = Vec::with_capacity(pending.len());
11391        let mut renudged: Vec<&str> = Vec::new();
11392        for &i in &pending {
11393            let src = &originals[i];
11394            // A seat now held by another agent than the job named has no
11395            // session of its own: it gets the full-context prompt whenever
11396            // it is asked in full (the handover ask, a nudge it cannot
11397            // resume).
11398            let full: &str = match &src.handover {
11399                Some(h) if specs[i].id != src.spec.id => h,
11400                _ => &src.prompt,
11401            };
11402            // The prompt and the budget are one decision: a nudge restates
11403            // finished work, a re-sent prompt redoes it.
11404            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11405                (
11406                    full.to_owned(),
11407                    src.timeout,
11408                    format!("{}-{word}-{}", src.stem, specs[i].id),
11409                )
11410            } else if nudges[i] == 0 {
11411                (full.to_owned(), src.timeout, src.stem.clone())
11412            } else {
11413                renudged.push(src.seat.key.as_str());
11414                let why = done[i]
11415                    .as_ref()
11416                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
11417                    .unwrap_or_else(|| "no parsable answer".to_owned());
11418                let nudge = prompt::nudge(&why);
11419                let nudged = has_context(&specs[i], &seats[i], src.sessions);
11420                let prompt = if nudged {
11421                    nudge
11422                } else {
11423                    format!("{full}\n\n---\n\n{nudge}")
11424                };
11425                (
11426                    prompt,
11427                    retry_budget(src.timeout, nudged),
11428                    format!("{}-retry{}", src.stem, nudges[i]),
11429                )
11430            };
11431            batch.push(SeatJob {
11432                spec: specs[i].clone(),
11433                seat: seats[i].clone(),
11434                cwd: src.cwd.clone(),
11435                prompt,
11436                timeout,
11437                allow_write: src.allow_write,
11438                sessions: src.sessions,
11439                artifacts: src.artifacts.clone(),
11440                stem,
11441                handover: None,
11442            });
11443        }
11444
11445        if !renudged.is_empty() {
11446            state.event(
11447                ctx.node,
11448                format!("retry {round}: re-asking {}", renudged.join(", ")),
11449            );
11450        }
11451        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11452        let mut still = Vec::new();
11453        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11454            seats[i] = seat;
11455            // No agent was asked: nothing to hand over, nudge, remember or
11456            // charge. The caller sees `resource_block` and stops the node.
11457            if let AgentOutcome::ResourceBlocked(why) = &out {
11458                done[i] = Some(Err(anyhow::anyhow!("{why}")));
11459                continue;
11460            }
11461            let class = FailClass::of(&out);
11462            // A dropped stream is nudged first (the conversation is still
11463            // there to pick up); only a seat whose nudges are spent hands over.
11464            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11465                && nudges[i] < retries
11466                && !roster.is_empty();
11467            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11468                let next = should_hand_over(prev[i].as_ref(), &cur)
11469                    .then(|| next_agent(i, &tried[i], &specs))
11470                    .flatten();
11471                if carry {
11472                    let h = state
11473                        .seat_history
11474                        .entry(originals[i].seat.key.clone())
11475                        .or_default();
11476                    h.failed.insert(specs[i].id.clone());
11477                    h.last_fail = Some(cur.clone());
11478                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11479                        h.last_ok = None;
11480                    }
11481                    // Saved before the next agent is asked, so a restart in
11482                    // between does not forget who failed.
11483                    if let Err(e) = state.save() {
11484                        tracing::warn!("could not persist a seat's failure history: {e:#}");
11485                    }
11486                }
11487                if let Some(next) = next {
11488                    record_handover(
11489                        state,
11490                        ctx.node,
11491                        &originals[i].seat.key,
11492                        &specs[i].id,
11493                        &next.id,
11494                        &cur,
11495                        &fail_reason(&out),
11496                    );
11497                    tried[i].insert(next.id.clone());
11498                    prev[i] = Some(cur.clone());
11499                    seats[i] =
11500                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11501                    specs[i] = next;
11502                    fresh[i] = Some(cur.stem_word().to_owned());
11503                    nudges[i] = 0;
11504                    done[i] = Some(Err(anyhow::anyhow!(
11505                        "handed over after: {}",
11506                        fail_reason(&out)
11507                    )));
11508                    still.push(i);
11509                    continue;
11510                }
11511            }
11512            if carry
11513                && !nudge_first
11514                && let Some(cur) = class.clone()
11515            {
11516                // The chain ended here (no successor, or a repeated failure).
11517                let h = state
11518                    .seat_history
11519                    .entry(originals[i].seat.key.clone())
11520                    .or_default();
11521                h.failed.insert(specs[i].id.clone());
11522                h.last_fail = Some(cur);
11523            }
11524            let parsed = match out {
11525                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11526                AgentOutcome::Quota(o) => {
11527                    last_quota[i] = o.quota.clone();
11528                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11529                }
11530                // Not a parseable answer: the nudge loop re-asks it, which is
11531                // exactly what a dropped stream needs. Just don't hand its raw
11532                // error JSON to `extract_json`.
11533                AgentOutcome::Dropped(o) => {
11534                    let why = o
11535                        .dropped
11536                        .as_ref()
11537                        .map(|d| d.why.as_str())
11538                        .unwrap_or("the CLI ended the stream without delivering its answer");
11539                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11540                }
11541                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11542                    Err(anyhow::anyhow!(e))
11543                }
11544            };
11545            let quota = class == Some(FailClass::Quota);
11546            let failed = parsed.is_err();
11547            if !failed && matches!(ctx.node, "judge" | "vote") {
11548                // A validated answer restores this seat's quorum presence,
11549                // including on the normal path after an all-seat block.
11550                state
11551                    .quota
11552                    .retain(|loss| loss.seat != originals[i].seat.key);
11553            }
11554            done[i] = Some(parsed);
11555            if carry && !failed {
11556                let h = state
11557                    .seat_history
11558                    .entry(originals[i].seat.key.clone())
11559                    .or_default();
11560                h.failed.remove(&specs[i].id);
11561                h.last_ok = Some(specs[i].id.clone());
11562                h.last_fail = None;
11563            }
11564            // Do not re-ask a rate-limited seat (quota) — a retry is known to
11565            // fail the same way; and never re-ask a seat that already parsed.
11566            // A failed agent that still has a successor is not re-asked
11567            // either: the handover was its remedy and has just been refused
11568            // (the chain stops on a repeated failure class). A seat with no
11569            // successor left (a single-agent roster, the roster's tail, or an
11570            // empty roster) keeps the same-agent nudge, bounded by `retries`.
11571            let agent_failure = class.is_some()
11572                && !nudge_first
11573                && !roster.is_empty()
11574                && next_agent(i, &tried[i], &specs).is_some();
11575            if failed && !quota && !agent_failure && nudges[i] < retries {
11576                nudges[i] += 1;
11577                still.push(i);
11578            }
11579        }
11580        pending = still;
11581    }
11582
11583    // One loss per seat whose chain ended on a quota: the intermediate ones
11584    // were absorbed by a handover and are not losses.
11585    for (i, q) in last_quota.into_iter().enumerate() {
11586        if let Some(quota) = q {
11587            losses.push(QuotaLoss::from_report(
11588                originals[i].seat.key.clone(),
11589                ctx.node,
11590                Some(&quota),
11591            ));
11592        }
11593    }
11594
11595    // A stalled quorum already has its own recovery checkpoint. Refresh
11596    // its missing seats before deciding whether it can resume.
11597    if state.status != RunStatus::Stalled
11598        && ctx.node != "deliberate"
11599        && n > 0
11600        && losses.len() == n
11601        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11602    {
11603        block_machine_outage(state, ctx.node, losses);
11604        state.quota.append(losses);
11605    }
11606
11607    seats
11608        .into_iter()
11609        .zip(done)
11610        .zip(nudges)
11611        .map(|((seat, res), attempts)| {
11612            (
11613                seat,
11614                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11615                attempts,
11616            )
11617        })
11618        .collect()
11619}
11620
11621/// A node with no available agent has no task verdict. Preserve the node
11622/// checkpoint and record why a subsequent explicit resume is needed.
11623pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11624    let mut auth: Vec<_> = losses
11625        .iter()
11626        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11627        .map(|loss| loss.agent.clone())
11628        .collect();
11629    auth.sort();
11630    auth.dedup();
11631    let (kind, detail) = if auth.is_empty() {
11632        (
11633            "agent usage limit",
11634            "All seats reached their agent usage limits; resume after usage is available"
11635                .to_owned(),
11636        )
11637    } else {
11638        (
11639            "agent authentication",
11640            format!(
11641                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11642                auth.join(", ")
11643            ),
11644        )
11645    };
11646    state.resource_block = Some(crate::run::ResourceBlock {
11647        node: node.to_owned(),
11648        kind: kind.to_owned(),
11649        cause: crate::run::ResourceBlockCause::Other,
11650        detail: detail.clone(),
11651        at: Timestamp::now(),
11652    });
11653    state.event(node, detail);
11654}
11655
11656/// Did the wave end without asking any agent (see [`block_wave`])?
11657fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11658    results
11659        .iter()
11660        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11661}
11662
11663/// Settle a wave that never spawned anything: every job answers
11664/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
11665/// else about the seat (session, history, quota) is touched.
11666fn block_wave(
11667    jobs: Vec<SeatJob>,
11668    state: &mut RunState,
11669    node: &str,
11670    attempt: usize,
11671    why: &str,
11672) -> Vec<(usize, SeatState, AgentOutcome)> {
11673    state.resource_wait = None;
11674    state
11675        .active
11676        .retain(|_, a| !(a.node == node && a.attempt == attempt));
11677    if let Err(e) = state.save() {
11678        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11679    }
11680    jobs.into_iter()
11681        .enumerate()
11682        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11683        .collect()
11684}
11685
11686/// Acquire the shared build cache's lease, waiting out contention within
11687/// `budget` (never past it — see AGENTS.md's build-cache section on why an
11688/// unbounded wait is never acceptable).
11689///
11690/// A first, non-blocking check happens before ever waiting; if it finds the
11691/// lease busy, that fact is logged as a `verify` event *and* flushed with
11692/// [`RunState::save`] immediately — not only once the wait finally succeeds
11693/// or gives up — so a `magi show` run by a different process while this one
11694/// is still waiting reads a `run.json` that says so, rather than whatever it
11695/// looked like before the wait started. The same applies to the terminal
11696/// failure: logged and saved before this returns `Err`, so a caller that
11697/// could not get the lease at all still leaves a legible record of why.
11698async fn acquire_cache_lease(
11699    state: &mut RunState,
11700    cache_dir: &Path,
11701    owner: &crate::cache::Owner,
11702    budget: Duration,
11703    node: &str,
11704    context: &str,
11705    cancel: &(dyn Fn() -> bool + Sync),
11706) -> Result<Option<crate::cache::Guard>> {
11707    let home = crate::run::home();
11708    let started = Instant::now();
11709    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11710        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11711        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11712        Err(e) => {
11713            state.event(
11714                "verify",
11715                format!("{context}: could not check the shared build cache: {e:#}"),
11716            );
11717            if let Err(e2) = state.save() {
11718                tracing::warn!("could not persist a cache-check failure: {e2:#}");
11719            }
11720            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11721        }
11722    };
11723    state.event(
11724        "verify",
11725        format!(
11726            "{context}: waiting for the shared build cache at {} ({})",
11727            cache_dir.display(),
11728            busy.describe()
11729        ),
11730    );
11731    // Seats of this node are waiting, not running: say so, so the countdown a
11732    // reader sees only ever measures an agent that has been spawned.
11733    state.resource_wait = Some(crate::run::ResourceWait {
11734        node: context.to_owned(),
11735        kind: "shared build cache".to_owned(),
11736        holder: busy.describe(),
11737        since: Timestamp::now(),
11738        limit_secs: budget.as_secs(),
11739    });
11740    if let Err(e) = state.save() {
11741        tracing::warn!("could not persist a cache wait: {e:#}");
11742    }
11743    let remaining = budget.saturating_sub(started.elapsed());
11744    let waited = crate::cache::wait_for_unless(
11745        &home,
11746        cache_dir,
11747        owner,
11748        remaining,
11749        Duration::from_secs(5),
11750        cancel,
11751    )
11752    .await;
11753    state.resource_wait = None;
11754    let cause = match &waited {
11755        Ok(Some(_)) => None,
11756        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11757        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11758            Some(crate::run::ResourceBlockCause::Timeout {
11759                waited_secs: started.elapsed().as_secs(),
11760                limit_secs: budget.as_secs(),
11761            })
11762        }
11763        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11764    };
11765    if let Some(cause) = cause {
11766        let detail = match &waited {
11767            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11768            Err(error) => format!("{error:#}"),
11769        };
11770        state.resource_block = Some(crate::run::ResourceBlock {
11771            cause,
11772            node: node.to_owned(),
11773            kind: "shared build cache".into(),
11774            detail,
11775            at: Timestamp::now(),
11776        });
11777        state.event(
11778            "resource",
11779            state.resource_block.as_ref().unwrap().description(),
11780        );
11781    }
11782    if let Err(error) = state.save() {
11783        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11784    }
11785    waited
11786}
11787
11788/// The lease state could not be read at all (unreadable lease file, a
11789/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11790/// known to be contended, so a wave degrades to running without a shared cache
11791/// instead of stopping on a hold that [`crate::triage`] could never release.
11792#[derive(Debug)]
11793struct CacheCheckError(String);
11794
11795impl std::fmt::Display for CacheCheckError {
11796    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11797        f.write_str(&self.0)
11798    }
11799}
11800
11801impl std::error::Error for CacheCheckError {}
11802
11803/// Keep a specific acquisition outcome when the caller adds command evidence.
11804fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11805    state
11806        .resource_block
11807        .get_or_insert_with(|| crate::run::ResourceBlock {
11808            cause: crate::run::ResourceBlockCause::Other,
11809            node: node.to_owned(),
11810            kind: "shared build cache".into(),
11811            detail: format!("{error:#}"),
11812            at: Timestamp::now(),
11813        });
11814    if let Err(error) = state.save() {
11815        tracing::warn!("could not persist cache resource block: {error:#}");
11816    }
11817}
11818
11819/// Run `body` — a verify command batch — while holding the shared build
11820/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11821/// never interleave with another borrower's build against the same
11822/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11823/// timeout, or a human's own `magi review`. See the `cache` module doc for
11824/// why this matters more than Cargo's own per-target locking covers — two
11825/// *different* worktrees building the same package name/version into one
11826/// cache directory is a staleness bug, not a lock contention one.
11827///
11828/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11829/// agent waves do. Repository scheduling also has its own bounded,
11830/// cancellable resource-wait policy. Contention must not consume the
11831/// verification budget: `body` receives the full `budget` after both leases
11832/// are acquired and the cache is fresh. Without a shared cache, only
11833/// repository coordination is acquired before handing over the full budget.
11834///
11835/// A lease that cannot be acquired within the resource wait is reported as a single
11836/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11837/// verification — the same shape a spawn failure already takes in
11838/// [`run_commands`], so a caller need not special-case it.
11839#[allow(clippy::too_many_arguments)]
11840async fn with_cache_lease<'s, F, Fut>(
11841    state: &'s mut RunState,
11842    cache_dir: Option<&Path>,
11843    node: &str,
11844    seat: &str,
11845    worktree: &Path,
11846    head: &str,
11847    budget: Duration,
11848    context: &str,
11849    wait: &crate::repo_lock::WaitPolicy,
11850    body: F,
11851) -> Result<(Vec<CommandOutcome>, bool)>
11852where
11853    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11854    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11855{
11856    // Match editing waves: cache first, repository second. In particular,
11857    // pre_gate must never hold the repository while waiting for a wave's cache.
11858    let started = Instant::now();
11859    let home = crate::run::home();
11860    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11861    let guard = if let Some(cache_dir) = cache_dir {
11862        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11863        match acquire_cache_lease(
11864            state,
11865            cache_dir,
11866            &owner,
11867            resource_limit,
11868            node,
11869            context,
11870            &|| wait.cancelled(),
11871        )
11872        .await
11873        .and_then(|g| {
11874            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11875        }) {
11876            Ok(g) => Some(g),
11877            Err(e) => {
11878                record_cache_error(state, node, &e);
11879                return Ok((
11880                    vec![CommandOutcome {
11881                        command: "(waiting for the shared build cache)".into(),
11882                        code: None,
11883                        output_tail: e.to_string(),
11884                        duration_ms: started.elapsed().as_millis() as u64,
11885                        resource_blocked: true,
11886                    }],
11887                    false,
11888                ));
11889            }
11890        }
11891    } else {
11892        None
11893    };
11894    let _coordination = coordinate_operation(state, worktree, wait).await?;
11895    if let Some(cache_dir) = cache_dir {
11896        let identity = crate::cache::Identity::new(worktree, head);
11897        if let Err(e) = crate::cache::ensure_fresh(
11898            &home,
11899            cache_dir,
11900            &identity,
11901            &state.config.verify.cargo_manifests(),
11902        ) {
11903            record_cache_error(state, node, &e);
11904            state.event(
11905                "verify",
11906                format!(
11907                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11908                    worktree.display(),
11909                    short(head)
11910                ),
11911            );
11912            if let Some(guard) = guard {
11913                guard.release();
11914            }
11915            return Ok((
11916                vec![CommandOutcome {
11917                    command: "(confirming the shared build cache is fresh)".into(),
11918                    code: None,
11919                    output_tail: e.to_string(),
11920                    duration_ms: started.elapsed().as_millis() as u64,
11921                    resource_blocked: true,
11922                }],
11923                false,
11924            ));
11925        }
11926    }
11927    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11928    if !timed_out_pids.is_empty() {
11929        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11930    }
11931    // Drop repository coordination before releasing the cache, reversing the
11932    // acquisition order. A cancelled/error future also drops both guards.
11933    drop(_coordination);
11934    if let Some(guard) = guard {
11935        guard.release();
11936    }
11937    Ok((outcomes, retried))
11938}
11939
11940/// Poll `pids` — commands [`run_commands`] reports as still running when its
11941/// own timeout elapsed — until every one is confirmed gone, or
11942/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11943///
11944/// Real confirmation where confirmation is possible, not a substitute for
11945/// full process-tree observation: a grandchild the timed-out process spawned
11946/// and that survives independently of it is invisible to a pid check the
11947/// same way it always was, and continuing to observe and collect *that*
11948/// stays a different piece of work with its own owner. This only narrows a
11949/// fixed blind wait into an actual check of the pids this process does know
11950/// about.
11951async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11952    wait_for_pids_with(
11953        pids,
11954        crate::proc::pid_alive,
11955        LEASE_RELEASE_POLL,
11956        LEASE_RELEASE_MAX_WAIT,
11957    )
11958    .await;
11959}
11960
11961/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
11962/// interval and ceiling supplied by the caller, so the polling *logic* -
11963/// returns as soon as every pid reports dead, gives up at the ceiling
11964/// otherwise - is testable on millisecond durations without asking the real
11965/// OS about a pid at all.
11966async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11967    pids: &[u32],
11968    alive: F,
11969    poll: Duration,
11970    max_wait: Duration,
11971) {
11972    let deadline = Instant::now() + max_wait;
11973    loop {
11974        if pids.iter().all(|&pid| !alive(pid)) {
11975            return;
11976        }
11977        if Instant::now() >= deadline {
11978            return;
11979        }
11980        tokio::time::sleep(poll).await;
11981    }
11982}
11983
11984/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
11985/// admission that it could not even get a verify command to run, as opposed
11986/// to evidence the command actually produced? A caller that would otherwise
11987/// read a resource-blocked outcome as a red command must check this first:
11988/// see [`Runner::gate`], which retries rather than records `Blocked` when
11989/// this is true.
11990fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11991    outcomes.iter().any(|o| o.resource_blocked)
11992}
11993
11994/// What [`Runner::gate_fix_round`] decided.
11995pub(crate) enum GateFix {
11996    /// The tree changed and `verify.e2e` is still green: run the gate again.
11997    Retry,
11998    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
11999    /// last failure stands and the run ends blocked.
12000    Stop,
12001    /// `verify.e2e` could not run after the fix (magi's own contention):
12002    /// decide nothing now, a later reentry retries.
12003    Defer,
12004}
12005
12006/// Is every red command in `outcomes` an ordinary failure the code could
12007/// explain: it ran, exited non-zero, and said something?
12008///
12009/// A timeout, a spawn failure and a killed process all leave `code` `None`;
12010/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
12011/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
12012/// what the command is stays the gate's business.
12013pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
12014    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12015    red.peek().is_some()
12016        && red.all(|o| {
12017            !o.resource_blocked
12018                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12019                && !o.output_tail.trim().is_empty()
12020        })
12021}
12022
12023/// Describe one verify command's outcome for the event log, distinguishing a
12024/// build/link failure — the toolchain never produced a binary to run — from
12025/// an actual test failure, since only the latter is a verdict on the patch.
12026fn e2e_outcome_label(o: &CommandOutcome) -> String {
12027    if o.ok() {
12028        return "pass".to_owned();
12029    }
12030    let reason = if o.build_failed() {
12031        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12032    } else {
12033        format!("FAIL ({:?})", o.code)
12034    };
12035    format!(
12036        "{reason}\n{}",
12037        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12038    )
12039}
12040
12041/// Run `verify.e2e`, retrying once if the first attempt could not build or
12042/// link — a build/link failure is frequently a race against a shared
12043/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
12044/// `verify` event per command, tagged with `context` (normally `"round N"`)
12045/// so the two call sites that need this — the ordinary per-round leg in
12046/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
12047/// it will ever call a round green — read identically in the event log.
12048async fn run_e2e_with_retry(
12049    state: &mut RunState,
12050    shell: &[String],
12051    commands: &[String],
12052    worktree: &Path,
12053    timeout: Duration,
12054    context: &str,
12055) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12056    let (mut e2e, mut timed_out_pids) = run_commands(
12057        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12058    )
12059    .await;
12060    for o in &e2e {
12061        state.event(
12062            "verify",
12063            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12064        );
12065    }
12066    // A build/link failure is not a verdict on the patch — it is frequently a
12067    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
12068    // one retry before letting a red like that decide the round.
12069    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12070    if verify_retried {
12071        state.event(
12072            "verify",
12073            format!(
12074                "{context}: verify could not build/link, not a test result — retrying once \
12075                 before concluding"
12076            ),
12077        );
12078        let retried = run_commands(
12079            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12080        )
12081        .await;
12082        e2e = retried.0;
12083        // Both attempts' timeouts matter, not just the last one: the first
12084        // attempt's descendants may still be alive alongside the retry's.
12085        timed_out_pids.extend(retried.1);
12086        for o in &e2e {
12087            state.event(
12088                "verify",
12089                format!(
12090                    "{context}: retry `{}` -> {}",
12091                    o.command,
12092                    e2e_outcome_label(o)
12093                ),
12094            );
12095        }
12096    }
12097    (e2e, verify_retried, timed_out_pids)
12098}
12099
12100/// Run configured shell commands in `cwd`, in order. The second element is
12101/// the pid of every command that hit `timeout` and was still running when
12102/// this stopped waiting on it (best-effort: `None` when the platform did not
12103/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
12104/// that releases a shared resource afterward needs to know.
12105///
12106/// Records `task` into [`RunState::active`] at every command boundary
12107/// (`RunState::task_command`) and clears it once the whole list has run
12108/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
12109/// for minutes with no seat and no output of its own to show for it (see
12110/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
12111/// apart from "not yet run" without this), and this is the only place that
12112/// knows which command is running right now and how many are left. Three
12113/// saves per command — start, not per second — matching the same "only at a
12114/// boundary" rule [`wave`] already follows for seats.
12115#[allow(clippy::too_many_arguments)]
12116async fn run_commands(
12117    state: &mut RunState,
12118    node: &str,
12119    task: &str,
12120    attempt: usize,
12121    shell: &[String],
12122    commands: &[String],
12123    cwd: &Path,
12124    timeout: Duration,
12125) -> (Vec<CommandOutcome>, Vec<u32>) {
12126    if commands.is_empty() {
12127        // Nothing to mark as running and nothing to clear — an empty list
12128        // means "not configured", and touching `active` (or the disk) over
12129        // that would be a write for every round of a repo with no
12130        // `verify.e2e` / `verify.gate` commands at all.
12131        return (Vec::new(), Vec::new());
12132    }
12133    let mut out = Vec::new();
12134    let mut timed_out_pids = Vec::new();
12135    let total = commands.len();
12136    for (idx, command) in commands.iter().enumerate() {
12137        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12138        if let Err(e) = state.save() {
12139            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12140        }
12141        let started = Instant::now();
12142        let mut cmd = tokio::process::Command::new(&shell[0]);
12143        cmd.quiet();
12144        // Verify the checkout as a standalone caller. Worker ownership and
12145        // pinned executable selection belong to this driver, not to tests or
12146        // tools it runs. Preserve home/seat context and Running's containment.
12147        for key in crate::worker::INVOCATION_ENV {
12148            cmd.env_remove(key);
12149        }
12150        cmd.args(&shell[1..])
12151            .arg(command)
12152            .current_dir(cwd)
12153            .stdin(std::process::Stdio::null())
12154            .stdout(std::process::Stdio::piped())
12155            .stderr(std::process::Stdio::piped())
12156            .kill_on_drop(true);
12157        let mut outputs = Vec::new();
12158        let protected = (|| -> Result<()> {
12159            crate::incremental::configure(&mut cmd, cwd)?;
12160            let mut paths = vec![cwd.to_owned()];
12161            if let Some(cache) = state.config.cache_dir() {
12162                paths.push(cache);
12163            }
12164            outputs.extend(crate::retention::OutputLease::admit(
12165                &crate::run::home(),
12166                &paths,
12167                state.config.disk.min_free_bytes,
12168            )?);
12169            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12170                outputs.push(guard);
12171            }
12172            if let Some(cache) = state.config.cache_dir() {
12173                outputs.push(crate::retention::OutputLease::cache(
12174                    &crate::run::home(),
12175                    &cache,
12176                    false,
12177                )?);
12178            }
12179            Ok(())
12180        })();
12181        if let Err(e) = protected {
12182            let detail = format!("execution admission unavailable: {e:#}");
12183            state.resource_block = Some(crate::run::ResourceBlock {
12184                cause: crate::run::ResourceBlockCause::Other,
12185                node: node.to_owned(),
12186                kind: if detail.contains("disk admission") {
12187                    "disk admission"
12188                } else {
12189                    "output ownership"
12190                }
12191                .to_owned(),
12192                detail: detail.clone(),
12193                at: Timestamp::now(),
12194            });
12195            out.push(CommandOutcome {
12196                command: command.clone(),
12197                code: None,
12198                output_tail: detail,
12199                duration_ms: 0,
12200                resource_blocked: true,
12201            });
12202            break;
12203        }
12204        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12205        let (code, body) = match spawned {
12206            Ok(mut running) => {
12207                let pid = running.child().id();
12208                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12209                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12210                let mut paths = vec![cwd.to_owned()];
12211                if let Some(cache) = state.config.cache_dir() {
12212                    paths.push(cache);
12213                }
12214                let mut pressure = None;
12215                let status = tokio::select! {
12216                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12217                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12218                        pressure = Some(reason); None
12219                    }
12220                };
12221                let (code, failure) = match status {
12222                    Some(Ok(Ok(status))) => (
12223                        status.code(),
12224                        status
12225                            .code()
12226                            .is_none()
12227                            .then(|| format!("command terminated: {status}")),
12228                    ),
12229                    Some(Ok(Err(error))) => {
12230                        (None, Some(format!("failed waiting for command: {error}")))
12231                    }
12232                    Some(Err(_)) => {
12233                        // Leased cleanup confirms process-tree death itself.
12234                        // Unleased commands still need the caller's PID check
12235                        // after kill-on-drop, before releasing cache ownership.
12236                        if !running.coordinated()
12237                            && let Some(pid) = pid
12238                        {
12239                            timed_out_pids.push(pid);
12240                        }
12241                        (
12242                            None,
12243                            Some(format!("timed out after {}s", timeout.as_secs())),
12244                        )
12245                    }
12246                    None => (None, None),
12247                };
12248                running.finish().await;
12249                if let Some(reason) = pressure {
12250                    state.resource_block = Some(crate::run::ResourceBlock {
12251                        cause: crate::run::ResourceBlockCause::Other,
12252                        node: node.to_owned(),
12253                        kind: "disk pressure".to_owned(),
12254                        detail: reason.clone(),
12255                        at: Timestamp::now(),
12256                    });
12257                    out.push(CommandOutcome {
12258                        command: command.clone(),
12259                        code: None,
12260                        output_tail: reason,
12261                        duration_ms: started.elapsed().as_millis() as u64,
12262                        resource_blocked: true,
12263                    });
12264                    break;
12265                }
12266                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12267                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12268                if let Some(failure) = failure {
12269                    body.push('\n');
12270                    body.push_str(&failure);
12271                }
12272                (code, body)
12273            }
12274            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12275        };
12276        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12277        if !body.is_empty() {
12278            // The full text is kept whatever the summary dropped. A fresh
12279            // name per command so a resume or retry never overwrites an
12280            // earlier run's output. A failed write changes no verdict.
12281            match crate::run::write_verification_artifact(state, task, &body) {
12282                Ok(name) if summary != body => {
12283                    summary = format!("[full output: artifacts/{name}]\n{summary}");
12284                }
12285                Ok(_) => {}
12286                Err(e) => {
12287                    let warning = format!("could not save the full {task} output: {e:#}");
12288                    tracing::warn!("{warning}");
12289                    state.event(node, warning);
12290                    if summary != body {
12291                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12292                    }
12293                }
12294            }
12295        }
12296        let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12297        out.push(CommandOutcome {
12298            command: command.clone(),
12299            code,
12300            output_tail: summary,
12301            duration_ms: started.elapsed().as_millis() as u64,
12302            resource_blocked,
12303        });
12304        if resource_blocked {
12305            state.resource_block = Some(crate::run::ResourceBlock {
12306                cause: crate::run::ResourceBlockCause::Other,
12307                node: node.to_owned(),
12308                kind: "incremental cache admission".into(),
12309                detail: body,
12310                at: Timestamp::now(),
12311            });
12312            break;
12313        }
12314    }
12315    state.task_finished(task);
12316    if let Err(e) = state.save() {
12317        tracing::warn!("could not persist the end of {task}: {e:#}");
12318    }
12319    (out, timed_out_pids)
12320}
12321
12322/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
12323/// section (`report::run`) and in the `merge` event this node records — for
12324/// the operator to run by hand.
12325///
12326/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
12327/// branch whose ruleset forbids merge commits (GitHub's "must not contain
12328/// merge commits", or "require linear history") rejects the push a `--no-ff`
12329/// merge would produce, which is exactly the guidance this function replaces.
12330/// `message`'s first line becomes the squash commit's subject, matching the
12331/// note `report::run` prints alongside this command — see that function for
12332/// why an explicit subject is not optional there.
12333fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12334    let repo = repo.display();
12335    match style {
12336        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12337        MergeStyle::Squash => {
12338            // The subject sits inside double quotes, and a title an agent
12339            // wrote may carry the characters that break out of them.
12340            let subject = message
12341                .lines()
12342                .next()
12343                .unwrap_or(branch)
12344                .replace(['\\', '"', '$', '`'], "");
12345            format!(
12346                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12347            )
12348        }
12349        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12350    }
12351}
12352
12353/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
12354/// under the hood, rejects a `title` over 256 characters and the whole
12355/// command fails — no PR at all, for a run whose body was otherwise fine
12356/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
12357/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
12358/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
12359/// this bound. It is a margin, not a guarantee — a title packed
12360/// with multi-unit characters could still in principle land close to the
12361/// edge, but a real task title's occasional emoji or accented letter fits
12362/// comfortably inside it.
12363const PR_TITLE_MAX: usize = 240;
12364
12365/// A pull request title from the opening line of `text`, or `None` when that
12366/// line is not English (GitHub text is) or has no letters.
12367///
12368/// A line within `max` is kept as is. A longer one is cut at the end of its
12369/// first sentence when that falls inside `max`, else at a word boundary with a
12370/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
12371/// make every merely-truncated title look non-English). The language check
12372/// runs on the kept text before any mark is added, so only what GitHub will
12373/// show is judged: an English opening followed by non-ASCII far past the cut
12374/// still passes.
12375fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12376    let line = queue::first_line(text)?;
12377    let chars: Vec<char> = line.chars().collect();
12378    let (kept, mark) = if chars.len() <= max {
12379        (line.to_owned(), "")
12380    } else if let Some(end) = sentence_end(&chars, max) {
12381        (chars[..end].iter().collect::<String>(), "")
12382    } else {
12383        let room = max.saturating_sub(3);
12384        // Cut at the last space inside the room; when the char just past the
12385        // room is a space the room already ends on a word.
12386        let cut = if chars[room].is_whitespace() {
12387            room
12388        } else {
12389            chars[..room]
12390                .iter()
12391                .rposition(|c| c.is_whitespace())
12392                .unwrap_or(room)
12393        };
12394        let head: String = chars[..cut].iter().collect();
12395        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12396        (head.to_owned(), "...")
12397    };
12398    if kept.is_empty() {
12399        return None;
12400    }
12401    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12402        return None;
12403    }
12404    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12405        return None;
12406    }
12407    Some(format!("{kept}{mark}"))
12408}
12409
12410/// The char length of the first sentence of `chars` when it ends within `max`
12411/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
12412/// abbreviations so `e.g. foo` does not end a title early.
12413fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12414    const MIN: usize = 20;
12415    for i in MIN..max.min(chars.len()) {
12416        if !matches!(chars[i], '.' | '!' | '?') {
12417            continue;
12418        }
12419        let Some(&next) = chars.get(i + 1) else {
12420            continue;
12421        };
12422        if !next.is_whitespace() {
12423            continue;
12424        }
12425        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12426        if after.is_some_and(|c| c.is_ascii_lowercase()) {
12427            continue;
12428        }
12429        let word: String = chars[..i]
12430            .iter()
12431            .rev()
12432            .take_while(|c| !c.is_whitespace())
12433            .collect::<Vec<_>>()
12434            .into_iter()
12435            .rev()
12436            .collect();
12437        let word = word.to_ascii_lowercase();
12438        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12439            continue;
12440        }
12441        let end = chars[..i]
12442            .iter()
12443            .rposition(|c| !c.is_whitespace())
12444            .map_or(i, |p| p + 1);
12445        return Some(end);
12446    }
12447    None
12448}
12449
12450/// What `merge = "pr"` (and the merge commit of the other modes) says about a
12451/// change: a title and a body describing what was *implemented*, not the task
12452/// that asked for it. A task reads as a request; a reader of the merged
12453/// history wants the change.
12454struct PrMessage {
12455    title: String,
12456    body: String,
12457}
12458
12459impl PrMessage {
12460    /// Title, blank line, body. The first line is the squash/merge commit
12461    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
12462    /// has to stay one sensible line.
12463    fn commit_message(&self) -> String {
12464        format!("{}\n\n{}", self.title, self.body)
12465    }
12466}
12467
12468/// The text after a leading `TITLE:` (any case) on `line`.
12469fn title_marker(line: &str) -> Option<&str> {
12470    let line = line.trim();
12471    let head = line.get(..6)?;
12472    head.eq_ignore_ascii_case("title:")
12473        .then(|| line[6..].trim())
12474}
12475
12476/// The implementer's own one-line title: the `TITLE:` line the implement
12477/// prompt asks for at the top of its SUMMARY. Candidate commits are all
12478/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
12479/// source, and a title that says as much is refused here too.
12480fn summary_title(summary: &str) -> Option<String> {
12481    let first = summary.lines().find(|l| !l.trim().is_empty())?;
12482    let raw = title_marker(first)?;
12483    if raw.is_empty() {
12484        return None;
12485    }
12486    let title = queue::title_from(raw, PR_TITLE_MAX);
12487    let lower = title.to_ascii_lowercase();
12488    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12489        return None;
12490    }
12491    Some(title)
12492}
12493
12494/// How `open_review`'s instruction begins; see [`landing_title`].
12495const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12496
12497/// Marker `open_review` gives a candidate that nothing in the roster wrote.
12498pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12499
12500/// Does this run review work that already existed, rather than implement a
12501/// task? Runs recorded before `reviewed_commits` existed carry only the
12502/// candidate marker.
12503pub(crate) fn is_review_run(state: &RunState) -> bool {
12504    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12505}
12506
12507/// The title of a review-only run: the subject of the oldest commit under
12508/// review. Later commits are usually fixups, and `instruction` is the review
12509/// prompt, which says nothing about the change. GitHub text is English, so a
12510/// non-ASCII or blank subject yields `None` and the caller's neutral title.
12511fn review_title(state: &RunState) -> Option<String> {
12512    english_subject(
12513        state.reviewed_commits.as_ref()?.first()?,
12514        state.config.graph.github_english(),
12515    )
12516}
12517
12518/// `raw` as a pull request title, or `None` when it is blank, not English
12519/// (GitHub text is), or one of magi's own candidate commit subjects.
12520fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12521    let raw = raw.trim();
12522    if raw.is_empty() {
12523        return None;
12524    }
12525    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12526        return None;
12527    }
12528    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12529        return None;
12530    }
12531    let title = queue::title_from(raw, PR_TITLE_MAX);
12532    let lower = title.to_ascii_lowercase();
12533    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12534        return None;
12535    }
12536    Some(title)
12537}
12538
12539/// What a review-only run's branch says about itself, read at the moment the
12540/// pull request is opened.
12541#[derive(Debug, Clone, PartialEq, Eq)]
12542struct BranchFacts {
12543    /// `(subject, body)` of each commit, oldest first.
12544    commits: Vec<(String, String)>,
12545    /// Trimmed `git diff --stat`.
12546    stat: String,
12547}
12548
12549/// Longest diff stat shown: this many file lines plus the summary line.
12550const STAT_FILE_LINES: usize = 25;
12551/// Cap on the commit list, well inside GitHub's 65536-character body limit.
12552const COMMITS_MAX_CHARS: usize = 20_000;
12553
12554/// Read the commits and diff stat of `base..branch`. `None` when git cannot
12555/// say or finds nothing, so the caller falls back to what the run recorded.
12556async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12557    let commits = git::commit_log(repo, base, branch).await.ok()?;
12558    if commits.is_empty() {
12559        return None;
12560    }
12561    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12562    let lines: Vec<&str> = stat.lines().collect();
12563    let stat = if lines.len() > STAT_FILE_LINES + 1 {
12564        let omitted = lines.len() - 1 - STAT_FILE_LINES;
12565        let more = format!(" ... {omitted} more file(s)");
12566        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12567        kept.push(&more);
12568        kept.push(lines[lines.len() - 1]);
12569        kept.join("\n")
12570    } else {
12571        lines.join("\n")
12572    };
12573    Some(BranchFacts { commits, stat })
12574}
12575
12576/// Defang what would break the surrounding markdown: a closing `</details>`
12577/// and a code fence.
12578fn markdown_safe(text: &str) -> String {
12579    text.replace("</details>", "&lt;/details&gt;")
12580        .replace("\x60\x60\x60", "~~~")
12581}
12582
12583fn neutral_title(state: &RunState, winner: char) -> String {
12584    format!(
12585        "chore: land candidate {} of run {}",
12586        winner.to_ascii_uppercase(),
12587        state.id
12588    )
12589}
12590
12591/// Reuse the graph's bounded fixer without starting another graph or driver.
12592/// Copy the working state back even on errors so spent rounds remain visible.
12593pub(crate) async fn repair_landing_gate(
12594    state: &mut RunState,
12595    winner: &Candidate,
12596    outcomes: &[CommandOutcome],
12597) -> Result<GateFix> {
12598    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12599        return Ok(GateFix::Stop);
12600    }
12601    let mut runner = Runner {
12602        roles: state.config.resolve_roles()?,
12603        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12604        state: state.clone(),
12605        pause: Pause::new(),
12606        interrupt: Pause::new(),
12607    };
12608    let result = runner.gate_fix_round(winner, outcomes).await;
12609    *state = runner.state;
12610    result
12611}
12612
12613/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
12614/// A completed failure is terminal on that SHA. Resource waits remain retryable.
12615pub(crate) async fn verify_current_head(
12616    state: &mut RunState,
12617    winner: &Candidate,
12618    wait: &crate::repo_lock::WaitPolicy,
12619) -> Result<bool> {
12620    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12621    // Preserve the previous record before replacing it, including on crash recovery.
12622    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12623        state
12624            .verification_history
12625            .push(crate::run::VerificationEvidence {
12626                superseded_by: Some(head.clone()),
12627                archived_at: Some(Timestamp::now()),
12628                fixes_spent: Some(state.gate_fixes.len()),
12629                gate_head: None,
12630                gate_ran: false,
12631                gate: Vec::new(),
12632                e2e: state.head_e2e.take(),
12633            });
12634    }
12635    if let Some(e) = &state.head_e2e
12636        && e.head == head
12637    {
12638        if e.passed {
12639            return Ok(true);
12640        }
12641        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12642            state.status = RunStatus::Blocked;
12643            state.save()?;
12644            return Ok(false);
12645        }
12646    } else if state.reviews.last().is_some_and(|r| {
12647        r.verified_head.as_deref() == Some(head.as_str())
12648            && (r.e2e_status() == E2eStatus::Passed
12649                || (r.e2e_status() == E2eStatus::NotConfigured
12650                    && state.config.verify.e2e.is_empty()))
12651    }) || state
12652        .gate_repair_e2e
12653        .as_ref()
12654        .is_some_and(|e| e.head == head && e.passed)
12655    {
12656        return Ok(true);
12657    }
12658    if let Some(e) = &state.gate_repair_e2e
12659        && e.head == head
12660        && !e.passed
12661        && !e.outcomes.is_empty()
12662        && !verify_inconclusive(&e.outcomes)
12663    {
12664        state.status = RunStatus::Blocked;
12665        state.save()?;
12666        return Ok(false);
12667    }
12668    state.head_e2e = Some(crate::run::GateRepairE2e {
12669        head: head.clone(),
12670        passed: false,
12671        outcomes: Vec::new(),
12672    });
12673    state.save()?;
12674    let commands = state.config.verify.e2e.clone();
12675    let shell = state.config.shell();
12676    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12677    let cache_dir = state.config.cache_dir();
12678    let outcomes = if commands.is_empty() {
12679        Vec::new()
12680    } else {
12681        let (outcomes, _) = with_cache_lease(
12682            state,
12683            cache_dir.as_deref(),
12684            "e2e",
12685            "e2e",
12686            &winner.worktree,
12687            &head,
12688            timeout,
12689            "current head e2e",
12690            wait,
12691            |state, budget| async move {
12692                run_e2e_with_retry(
12693                    state,
12694                    &shell,
12695                    &commands,
12696                    &winner.worktree,
12697                    budget,
12698                    "current head e2e",
12699                )
12700                .await
12701            },
12702        )
12703        .await?;
12704        outcomes
12705    };
12706    let unresolved = verify_inconclusive(&outcomes);
12707    let passed = !unresolved
12708        && outcomes.iter().all(CommandOutcome::ok)
12709        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12710    if !outcomes.is_empty() && !unresolved {
12711        state.record_base_progress(&head);
12712    }
12713    state.head_e2e = Some(crate::run::GateRepairE2e {
12714        head: head.clone(),
12715        passed,
12716        outcomes,
12717    });
12718    if !passed && !unresolved {
12719        state.status = RunStatus::Blocked;
12720        state.event(
12721            "e2e",
12722            format!("verify.e2e failed on head {head}; nothing published"),
12723        );
12724    }
12725    state.save()?;
12726    Ok(passed)
12727}
12728
12729/// Recheck a repair's e2e requirement across failures, resource deferrals and
12730/// process interruptions. A successful result only applies to its exact SHA.
12731pub(crate) async fn check_gate_repair_e2e(
12732    state: &mut RunState,
12733    winner: &Candidate,
12734    wait: &crate::repo_lock::WaitPolicy,
12735) -> Result<GateFix> {
12736    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12737        return Ok(GateFix::Retry);
12738    }
12739    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12740    if state
12741        .gate_repair_e2e
12742        .as_ref()
12743        .is_some_and(|r| r.head == head && r.passed)
12744    {
12745        return Ok(GateFix::Retry);
12746    }
12747    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12748        head: head.clone(),
12749        passed: false,
12750        outcomes: Vec::new(),
12751    });
12752    state.save()?;
12753    let commands = state.config.verify.e2e.clone();
12754    let outcomes = if commands.is_empty() {
12755        Vec::new()
12756    } else {
12757        let shell = state.config.shell();
12758        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12759        let cache_dir = state.config.cache_dir();
12760        let (outcomes, _) = with_cache_lease(
12761            state,
12762            cache_dir.as_deref(),
12763            "e2e",
12764            "e2e",
12765            &winner.worktree,
12766            &head,
12767            timeout,
12768            "gate repair e2e",
12769            wait,
12770            |state, budget| async move {
12771                run_e2e_with_retry(
12772                    state,
12773                    &shell,
12774                    &commands,
12775                    &winner.worktree,
12776                    budget,
12777                    "gate repair e2e",
12778                )
12779                .await
12780            },
12781        )
12782        .await?;
12783        outcomes
12784    };
12785    let deferred = verify_inconclusive(&outcomes);
12786    let passed = outcomes.iter().all(CommandOutcome::ok)
12787        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12788    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12789        head,
12790        passed,
12791        outcomes,
12792    });
12793    state.save()?;
12794    if deferred {
12795        Ok(GateFix::Defer)
12796    } else if !passed {
12797        state.event(
12798            "gate",
12799            "verify.e2e failed after gate repair; nothing published",
12800        );
12801        Ok(GateFix::Stop)
12802    } else {
12803        Ok(GateFix::Retry)
12804    }
12805}
12806
12807/// Run the same configured gate for a landing head and a fresh graph.
12808pub(crate) async fn run_landing_gate(
12809    state: &mut RunState,
12810    winner: &Candidate,
12811    wait: &crate::repo_lock::WaitPolicy,
12812) -> Result<Vec<CommandOutcome>> {
12813    let shell = state.config.shell();
12814    let gate_commands = state.config.verify.gate.clone();
12815    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12816    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12817        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12818        Vec::new()
12819    } else {
12820        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12821        let cache_dir = state.config.cache_dir();
12822        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12823        let (outcomes, _) = with_cache_lease(
12824            state,
12825            cache_dir.as_deref(),
12826            "gate",
12827            "gate",
12828            &winner.worktree,
12829            &head,
12830            timeout,
12831            "final gate",
12832            wait,
12833            |state, budget| {
12834                let shell = shell.clone();
12835                let gate_commands = gate_commands.clone();
12836                let winner = winner.clone();
12837                async move {
12838                    Runner::run_pre_gate(state, &winner).await;
12839                    // Bind command evidence after mechanical fixes, before commands start.
12840                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12841                    let (outcomes, timed_out_pids) = run_commands(
12842                        state,
12843                        "gate",
12844                        "gate",
12845                        0,
12846                        &shell,
12847                        &gate_commands,
12848                        &winner.worktree,
12849                        budget,
12850                    )
12851                    .await;
12852                    (outcomes, false, timed_out_pids)
12853                }
12854            },
12855        )
12856        .await?;
12857        outcomes
12858    };
12859    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12860    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12861        let mut outcomes = outcomes;
12862        outcomes.push(CommandOutcome {
12863            command: "gate head stability".into(), code: None,
12864            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12865            duration_ms: 0, resource_blocked: false,
12866        });
12867        return Ok(outcomes);
12868    }
12869    if outcomes.is_empty() {
12870        // Nothing configured to check — distinct from every other
12871        // silence in this run's event log, since an empty `gate` alone
12872        // no longer says whether the gate ran at all (see
12873        // `RunState::gate_ran`'s own doc).
12874        state.event("gate", "no gate commands configured; not tested");
12875    }
12876    for o in &outcomes {
12877        state.event(
12878            "gate",
12879            format!(
12880                "`{}` -> {}",
12881                o.command,
12882                if o.ok() {
12883                    "pass".to_owned()
12884                } else {
12885                    format!(
12886                        "FAIL ({:?})\n{}",
12887                        o.code,
12888                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12889                    )
12890                }
12891            ),
12892        );
12893    }
12894    Ok(outcomes)
12895}
12896
12897/// The pull request title to hand to `land::merge_subject`. A review-only run
12898/// opened by an earlier build titled its pull request with the review prompt;
12899/// that title is dropped (empty, so the fallback applies) rather than landed.
12900/// Any other title, including an operator's rename, passes through untouched,
12901/// and so does every title of a run that implements a task.
12902pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12903    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12904        ""
12905    } else {
12906        pr_title
12907    }
12908}
12909
12910/// What the squash subject falls back to when the pull request title is empty
12911/// or candidate-shaped: for a review-only run the derived title, never the
12912/// review prompt held in `instruction`.
12913pub fn landing_subject_source(state: &RunState) -> String {
12914    if is_review_run(state) {
12915        let winner = state.candidates.first().map_or('A', |c| c.label);
12916        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12917    }
12918    state.instruction.clone()
12919}
12920
12921/// `summary` without its `TITLE:` line, which the pull request title already
12922/// carries.
12923fn summary_without_title(summary: &str) -> String {
12924    let mut lines = summary.trim().lines().peekable();
12925    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12926        lines.next();
12927    }
12928    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12929}
12930
12931/// The pull request title and body for the winning candidate.
12932///
12933/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
12934/// the task's own opening line via [`queue::title_from`] when there is none.
12935/// `state.instruction` can open with blank lines (`task_text` only rejects a
12936/// body that is blank *entirely*), which `title_from` skips.
12937///
12938/// Publishing uses an independent description generated from the final diff
12939/// and structured evidence. Internal summaries and reports remain local.
12940/// Non-publishing previews may display the original task and candidate summary.
12941/// The footer retains the run and candidate provenance.
12942#[cfg(test)]
12943fn pr_message(state: &RunState, winner: char) -> PrMessage {
12944    pr_message_with(state, winner, None)
12945}
12946
12947/// [`pr_message`] with what the branch of a review-only run says about itself.
12948/// `facts` is ignored for a run that implements a task.
12949#[cfg(test)]
12950fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12951    let raw = pr_message_raw(
12952        state,
12953        winner,
12954        facts,
12955        crate::pr_outcome::FinalHead::Unreported,
12956        ChangeNote::NotAsked,
12957    );
12958    let id = crate::scrub::Identity::current();
12959    PrMessage {
12960        title: crate::scrub::scrub(&raw.title, &id),
12961        body: crate::scrub::scrub(&raw.body, &id),
12962    }
12963}
12964
12965/// What the public description may say about the change itself.
12966#[derive(Debug, Clone, Copy)]
12967enum ChangeNote<'a> {
12968    /// A local preview or non-publishing merge mode: no public generation needed.
12969    NotAsked,
12970    /// A summary written from the final diff.
12971    Summary(&'a crate::run::ChangeSummary),
12972    /// A summary was asked for and could not be produced; the text says why.
12973    Unavailable(&'a str),
12974}
12975
12976fn pr_message_raw(
12977    state: &RunState,
12978    winner: char,
12979    facts: Option<&BranchFacts>,
12980    head: crate::pr_outcome::FinalHead<'_>,
12981    change: ChangeNote<'_>,
12982) -> PrMessage {
12983    if !matches!(change, ChangeNote::NotAsked) {
12984        let (title, mut body) = match change {
12985            ChangeNote::Summary(c) => (
12986                c.title.clone(),
12987                format!("## Summary\n\n{}\n\n", c.body.trim()),
12988            ),
12989            ChangeNote::Unavailable(why) => (
12990                neutral_title(state, winner),
12991                format!(
12992                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12993                ),
12994            ),
12995            ChangeNote::NotAsked => unreachable!(),
12996        };
12997        body.push_str(&public_pr_evidence(state, head));
12998        body.push_str(&format!(
12999            "\n---\nmagi:run/{} magi:candidate-{}\n",
13000            state.id,
13001            winner.to_ascii_lowercase()
13002        ));
13003        return PrMessage { title, body };
13004    }
13005    pr_preview(state, winner, facts, head)
13006}
13007
13008/// Append public facts from the run records after every prose generation,
13009/// including posting-gate rewrites. The agent cannot drop this provenance.
13010fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
13011    let mut body = crate::pr_outcome::review_rounds(state);
13012    body.push_str(&crate::pr_outcome::final_verification(state, head));
13013    let open = state.last_round_findings();
13014    if !open.is_empty() {
13015        body.push_str("## Open review findings\n\n");
13016        for f in open {
13017            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13018        }
13019        body.push('\n');
13020    }
13021    if let Some(c) = state.change_summary.as_ref()
13022        && head == crate::pr_outcome::FinalHead::At(&c.head)
13023    {
13024        body.push_str(&format!(
13025            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13026            c.base, c.head
13027        ));
13028    }
13029    body
13030}
13031
13032/// Local-only preview of the operator's task and candidate summary.
13033fn pr_preview(
13034    state: &RunState,
13035    winner: char,
13036    facts: Option<&BranchFacts>,
13037    head: crate::pr_outcome::FinalHead<'_>,
13038) -> PrMessage {
13039    let summary = state
13040        .candidates
13041        .iter()
13042        .find(|c| c.label == winner)
13043        .map(|c| c.summary.as_str())
13044        .unwrap_or_default();
13045    // The fallback is the operator's own words and may not be English; GitHub
13046    // text always is, so a non-English task gets a neutral title instead.
13047    let review = is_review_run(state);
13048    let title = if review {
13049        facts
13050            .and_then(|f| {
13051                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13052            })
13053            .or_else(|| review_title(state))
13054            .or_else(|| {
13055                state
13056                    .candidates
13057                    .iter()
13058                    .find(|c| c.label == winner)
13059                    .filter(|c| !c.branch.starts_with("magi/"))
13060                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13061            })
13062            .unwrap_or_else(|| neutral_title(state, winner))
13063    } else {
13064        summary_title(summary).unwrap_or_else(|| {
13065            english_title(
13066                &state.instruction,
13067                PR_TITLE_MAX,
13068                state.config.graph.github_english(),
13069            )
13070            .unwrap_or_else(|| neutral_title(state, winner))
13071        })
13072    };
13073
13074    let mut body = String::new();
13075    let what = summary_without_title(summary);
13076    if !what.is_empty() {
13077        body.push_str("## Summary\n\n");
13078        body.push_str(&what);
13079        body.push_str("\n\n");
13080    }
13081
13082    // Review history uses structured facts across all rounds, never reports.
13083    body.push_str(&crate::pr_outcome::review_rounds(state));
13084    body.push_str(&crate::pr_outcome::final_verification(state, head));
13085    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13086
13087    let open = state.open_findings();
13088    if !open.is_empty() {
13089        body.push_str("## Open review findings\n\n");
13090        for f in &open {
13091            body.push_str(&format!(
13092                "- `{}` [{:?}] {}\n",
13093                f.id, f.severity, "Context retained in local review records."
13094            ));
13095        }
13096        body.push('\n');
13097    }
13098
13099    if let Some(fix) = fix
13100        && !fix.rejected.is_empty()
13101    {
13102        body.push_str("## Declined by the fixer\n\n");
13103        for r in &fix.rejected {
13104            body.push_str(&format!(
13105                "- `{}`: {}\n",
13106                r.id, "Reason retained in local review records."
13107            ));
13108        }
13109        body.push('\n');
13110    }
13111
13112    if review {
13113        // The review prompt is not the task; list what the branch carries.
13114        body.push_str("## Commits under review\n\n");
13115        if let Some(facts) = facts {
13116            let mut left = COMMITS_MAX_CHARS;
13117            for (i, (subject, text)) in facts.commits.iter().enumerate() {
13118                let mut entry = format!("- {}\n", markdown_safe(subject));
13119                for l in markdown_safe(text).lines() {
13120                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
13121                }
13122                if left == 0 {
13123                    body.push_str(&format!(
13124                        "- ... {} more commit(s)\n",
13125                        facts.commits.len() - i
13126                    ));
13127                    break;
13128                }
13129                if entry.len() > left {
13130                    // Even the first commit is cut: one huge body must not
13131                    // push the whole description past GitHub's limit.
13132                    let mut end = left;
13133                    while !entry.is_char_boundary(end) {
13134                        end -= 1;
13135                    }
13136                    entry.truncate(end);
13137                    entry.push_str("\n  ... (truncated)\n");
13138                    left = 0;
13139                } else {
13140                    left -= entry.len();
13141                }
13142                body.push_str(&entry);
13143            }
13144            if !facts.stat.trim().is_empty() {
13145                body.push_str(&format!(
13146                    "\n## Diff stat\n\n```\n{}\n```\n",
13147                    markdown_safe(facts.stat.trim())
13148                ));
13149            }
13150        } else {
13151            match &state.reviewed_commits {
13152                Some(subjects) => {
13153                    for s in subjects {
13154                        body.push_str(&format!("- {}\n", s.trim()));
13155                    }
13156                }
13157                None => {
13158                    // An older run kept only the prompt, with the commit list
13159                    // after its first paragraph.
13160                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13161                    body.push_str(rest.trim());
13162                    body.push('\n');
13163                }
13164            }
13165        }
13166    } else {
13167        let task = state.instruction.trim();
13168        let task = if task.is_empty() {
13169            "(empty task)"
13170        } else {
13171            task
13172        };
13173        body.push_str(&format!(
13174            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13175            task.replace("</details>", "&lt;/details&gt;")
13176        ));
13177    }
13178
13179    body.push_str(&format!(
13180        "\n---\nmagi:run/{} magi:candidate-{}\n",
13181        state.id,
13182        winner.to_ascii_lowercase()
13183    ));
13184
13185    PrMessage { title, body }
13186}
13187
13188/// Source imports have already been prepared from structured inputs. Legacy
13189/// inferred references must never instruct a seat to reconstruct those imports.
13190fn seeded_instruction(state: &RunState) -> String {
13191    let mut inputs = Vec::new();
13192    if let Some(start) = &state.sources.start_from {
13193        inputs.push(format!(
13194            "Start from `{}` at `{}` (including ancestry).",
13195            start.token, start.sha
13196        ));
13197    }
13198    for pick in &state.sources.pick_commits {
13199        inputs.push(format!(
13200            "Pick `{}` at `{}` (single commit).",
13201            pick.token, pick.sha
13202        ));
13203    }
13204    if inputs.is_empty() {
13205        state.instruction.clone()
13206    } else {
13207        format!(
13208            "{}\n\n# Explicit source inputs already prepared\n\n{}",
13209            state.instruction,
13210            inputs.join("\n")
13211        )
13212    }
13213}
13214
13215/// Does the winner have no commits ahead of the base it would land on?
13216/// Any failure to find out reads as "not empty": the merge then behaves as it
13217/// always did rather than refusing on a guess.
13218async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13219    let base = &state.base_branch;
13220    let mut against = base.clone();
13221    if mode == MergeMode::Pr {
13222        // A pull request lands on the remote's base, never the local branch
13223        // of the same name: if that cannot be read, "not empty" is the safe
13224        // answer.
13225        let remote = &state.config.merge.remote;
13226        let tracking = format!("{remote}/{base}");
13227        let fetched = git::fetch(repo, remote, base).await;
13228        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13229            against = tracking;
13230        } else {
13231            return false;
13232        }
13233    }
13234    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13235}
13236
13237/// Why nothing was opened for an empty winner, with what the task's own
13238/// references resolved to.
13239fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13240    let mut detail = format!(
13241        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13242         nothing to open a pull request for"
13243    );
13244    detail.push_str(
13245        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13246    );
13247    detail
13248}
13249
13250/// What the `Pr` merge does once it knows whether the branch already has an
13251/// open pull request.
13252#[derive(Debug, PartialEq, Eq)]
13253enum PrPlan {
13254    Create,
13255    Adopt { url: String, title: String },
13256    Stop(String),
13257}
13258
13259/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
13260/// a failed lookup -> stop with the real reason. Never guesses.
13261fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13262    match found {
13263        Ok(land::OpenPr::None) => PrPlan::Create,
13264        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13265        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13266            "several open pull requests exist for this branch, not picking one: {}",
13267            urls.join(" ")
13268        )),
13269        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13270    }
13271}
13272
13273/// `gh pr create`, returning the PR url.
13274pub(crate) async fn gh_pr_create(
13275    cwd: &Path,
13276    base: &str,
13277    head: &str,
13278    title: &str,
13279    body: &str,
13280) -> Result<String> {
13281    let out = tokio::process::Command::new("gh")
13282        .args([
13283            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13284        ])
13285        .current_dir(cwd)
13286        .quiet()
13287        .stdin(std::process::Stdio::null())
13288        .output()
13289        .await
13290        .context("spawn gh")?;
13291    if out.status.success() {
13292        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13293    } else {
13294        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13295    }
13296}
13297
13298/// Never force deletion of uncommitted work, including unknown legacy trees.
13299async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13300    match std::fs::symlink_metadata(path) {
13301        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13302            // A previous cleanup may have removed the tree before a branch
13303            // deletion or state save failed. Unregister it if still present.
13304            git::worktree_remove_clean(repo, path).await?;
13305            return Ok(());
13306        }
13307        Err(e) => return Err(e.into()),
13308        Ok(_) => {}
13309    }
13310    if !git::is_clean(path).await? {
13311        bail!(
13312            "worktree has uncommitted work; refusing removal: {}",
13313            path.display()
13314        );
13315    }
13316    if !git::worktree_remove_clean(repo, path).await? {
13317        bail!("worktree removal refused: {}", path.display());
13318    }
13319    Ok(())
13320}
13321
13322/// Tear a run's worktrees and branches down.
13323///
13324/// `home` is where the updated `run.json` is saved (via
13325/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
13326/// a housekeeping pass already has its own honest `home` handed to it, and
13327/// falling through to the global here would write back through whichever
13328/// directory some other process or test pinned into that `OnceLock` first,
13329/// not the one the caller actually resolved its `runs` and `state` from.
13330pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13331    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13332        bail!("refusing fold: {reason}");
13333    }
13334    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13335        bail!("candidate preservation incomplete; refusing cleanup");
13336    }
13337    // Serialize ordinary folding with the same starts/resumes as target-only
13338    // eviction. Otherwise a second janitor could remove the whole worktree
13339    // around a live output borrower even though target eviction was fenced.
13340    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13341    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13342    let _driver = crate::driver::maintenance(home, &state.id)?;
13343    let mut output_guards = Vec::new();
13344    for entry in std::fs::read_dir(state.worktree_root())
13345        .into_iter()
13346        .flatten()
13347        .flatten()
13348    {
13349        if entry.path().join(".git").is_file() {
13350            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13351        }
13352    }
13353    let repo = state.repo.clone();
13354    // CLI, Web and automatic cleanup all enter here, independently of
13355    // Runner::fold_losers. Protect deletion and shared config release too.
13356    let wait = repository_wait(state, None);
13357    let _coordination = match coordinate_operation(state, &repo, &wait).await {
13358        Ok(lease) => lease,
13359        Err(e) => {
13360            state.save_under(home)?;
13361            return Err(e);
13362        }
13363    };
13364    // Confirmed merges dispose of their owned sources after durable evidence
13365    // capture. Preflight every source before removing anything.
13366    let _source_outputs = if state.status == RunStatus::Merged
13367        && state.merge.as_ref().is_some_and(|merge| merge.ok)
13368    {
13369        verify_merged_fold_sources(state, drop_winner, home).await?
13370    } else {
13371        Vec::new()
13372    };
13373    let root = state.worktree_root();
13374    let winner = state.tally.as_ref().map(|t| t.winner);
13375    let mut removed = Vec::new();
13376
13377    for i in 0..state.candidates.len() {
13378        let c = state.candidates[i].clone();
13379        let is_winner = Some(c.label) == winner;
13380        if is_winner && !drop_winner {
13381            continue;
13382        }
13383        let existed = c.worktree.exists();
13384        // A vanished directory can still have a Git registration that keeps
13385        // its branch checked out. Remove that registration before the branch.
13386        remove_preserved_worktree(&repo, &c.worktree).await?;
13387        if existed {
13388            removed.push(c.worktree.to_string_lossy().into_owned());
13389        }
13390        // A branch handed to a later run (and its pull request) is not this
13391        // run's to delete.
13392        let handed_over = state.released_branches.contains(&c.branch);
13393        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13394            if !git::branch_delete(&repo, &c.branch).await? {
13395                bail!("candidate branch removal refused");
13396            }
13397            removed.push(c.branch.clone());
13398        }
13399        state.candidates[i].folded = true;
13400    }
13401
13402    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13403        let path = name.path();
13404        let keep = !drop_winner
13405            && winner.is_some_and(|w| {
13406                path.file_name()
13407                    .is_some_and(|n| n == format!("cand-{w}").as_str())
13408            });
13409        if keep {
13410            continue;
13411        }
13412        remove_preserved_worktree(&repo, &path).await?;
13413        removed.push(path.to_string_lossy().into_owned());
13414    }
13415
13416    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
13417    // judge worktrees, so once the loop above has cleared all of them out,
13418    // the parent is a bare directory nobody else was ever going to remove -
13419    // git only ever managed what was inside it. Left alone, one of these
13420    // accumulates per fully-folded run; the operator's own machine had 74.
13421    // `remove_if_empty` re-checks rather than assuming: a run whose winner
13422    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
13423    // so does anything a run never claimed that happens to share the bay.
13424    remove_if_empty(&root);
13425
13426    if state.enabled_worktree_config && drop_winner {
13427        // A release, not a raw disable: some sibling run in this repository
13428        // may still hold its own reference (see `git::acquire_worktree_config`),
13429        // and only the last release actually turns the setting back off.
13430        git::release_worktree_config(&repo).await.ok();
13431        state.enabled_worktree_config = false;
13432    }
13433    state.save_under(home)?;
13434    Ok(removed)
13435}
13436
13437async fn verify_merged_fold_sources(
13438    state: &RunState,
13439    drop_winner: bool,
13440    home: &Path,
13441) -> Result<Vec<std::fs::File>> {
13442    let winner = state.tally.as_ref().map(|t| t.winner);
13443    let mut heads = BTreeSet::new();
13444    for candidate in &state.candidates {
13445        if Some(candidate.label) == winner && !drop_winner {
13446            continue;
13447        }
13448        anyhow::ensure!(
13449            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13450            "candidate recovery source is outside the inventoried run root: {}",
13451            candidate.worktree.display()
13452        );
13453        match std::fs::symlink_metadata(&candidate.worktree) {
13454            Ok(_) => {}
13455            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13456            Err(error) => return Err(error.into()),
13457        }
13458        if !state.released_branches.contains(&candidate.branch)
13459            && git::branch_exists(&state.repo, &candidate.branch).await?
13460        {
13461            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13462        }
13463    }
13464    let mut cargo_guards = Vec::new();
13465    let entries = match std::fs::read_dir(state.worktree_root()) {
13466        Ok(entries) => Some(entries),
13467        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13468        Err(error) => return Err(error.into()),
13469    };
13470    for entry in entries.into_iter().flatten() {
13471        let path = entry?.path();
13472        let metadata = std::fs::symlink_metadata(&path)?;
13473        anyhow::ensure!(
13474            metadata.is_dir() && !metadata.file_type().is_symlink(),
13475            "non-directory or symlink recovery source retained: {}",
13476            path.display()
13477        );
13478        if !drop_winner
13479            && winner.is_some_and(|w| {
13480                path.file_name()
13481                    .is_some_and(|name| name == format!("cand-{w}").as_str())
13482            })
13483        {
13484            continue;
13485        }
13486        anyhow::ensure!(
13487            path.join(".git").is_file(),
13488            "unregistered recovery source retained: {}",
13489            path.display()
13490        );
13491        anyhow::ensure!(
13492            git::git(
13493                &path,
13494                &[
13495                    "status",
13496                    "--porcelain",
13497                    "--untracked-files=all",
13498                    "--ignored",
13499                    "--",
13500                    ".",
13501                    ":(exclude)target"
13502                ]
13503            )
13504            .await?
13505            .is_empty(),
13506            "source changes or ignored recovery files retained: {}",
13507            path.display()
13508        );
13509        match std::fs::symlink_metadata(path.join("target")) {
13510            Ok(_) => {
13511                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13512            }
13513            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13514            Err(error) => return Err(error.into()),
13515        }
13516        for checkpoint in [
13517            "rebase-merge",
13518            "rebase-apply",
13519            "sequencer",
13520            "CHERRY_PICK_HEAD",
13521            "MERGE_HEAD",
13522            "REBASE_HEAD",
13523            "REVERT_HEAD",
13524            "BISECT_LOG",
13525            "AUTO_MERGE",
13526            "MERGE_MSG",
13527        ] {
13528            let location = git::git(
13529                &path,
13530                &[
13531                    "rev-parse",
13532                    "--path-format=absolute",
13533                    "--git-path",
13534                    checkpoint,
13535                ],
13536            )
13537            .await?;
13538            anyhow::ensure!(
13539                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13540                if e.kind() == std::io::ErrorKind::NotFound),
13541                "Git recovery checkpoint retained: {checkpoint}"
13542            );
13543        }
13544        heads.insert(git::rev_parse(&path, "HEAD").await?);
13545    }
13546    crate::fold_evidence::preserve(state, home, &heads).await?;
13547
13548    Ok(cargo_guards)
13549}
13550
13551/// Remove `dir` if it exists and has nothing in it.
13552///
13553/// Best-effort and silent by design: a directory that is not empty (a run
13554/// whose winner is still parked there, a stray file some other process left)
13555/// is exactly the case this must refuse, and a directory that is already gone
13556/// is not a failure worth reporting either. `std::fs::remove_dir` itself
13557/// already refuses a non-empty directory, so the emptiness check below is
13558/// belt, not suspenders - it is what keeps this from ever attempting the
13559/// removal in the case that matters, rather than trusting `remove_dir`'s
13560/// error path to have no side effects if it ever changed.
13561fn remove_if_empty(dir: &Path) {
13562    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13563        std::fs::remove_dir(dir).ok();
13564    }
13565}
13566
13567/// Severity of the worst open finding in the last review round, for reporting.
13568pub fn worst_open(state: &RunState) -> Option<Severity> {
13569    state
13570        .reviews
13571        .last()?
13572        .reviews
13573        .iter()
13574        .flat_map(|r| r.findings.iter())
13575        .map(|f| f.severity)
13576        .max()
13577}
13578
13579/// `[worktree] setup` hides its products per worktree, which needs
13580/// `extensions.worktreeConfig`. Taken once per run and released with the run's
13581/// worktrees, like the hook's; every place that runs setup calls this first.
13582async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13583    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13584        git::acquire_worktree_config(repo).await?;
13585        state.enabled_worktree_config = true;
13586    }
13587    Ok(())
13588}
13589
13590#[cfg(test)]
13591mod tests {
13592    #[tokio::test]
13593    async fn launcher_base_fetch_requires_remote_success() {
13594        let tmp = tempfile::tempdir().unwrap();
13595        let repo = tmp.path().join("repo");
13596        let remote = tmp.path().join("remote");
13597        std::fs::create_dir(&repo).unwrap();
13598        std::fs::create_dir(&remote).unwrap();
13599        git::git(&remote, &["init", "--bare", "-b", "main"])
13600            .await
13601            .unwrap();
13602        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13603        git::git(
13604            &repo,
13605            &[
13606                "-c",
13607                "user.name=Fixture",
13608                "-c",
13609                "user.email=fixture@example.invalid",
13610                "commit",
13611                "--allow-empty",
13612                "-m",
13613                "fixture",
13614            ],
13615        )
13616        .await
13617        .unwrap();
13618        git::git(
13619            &repo,
13620            &["remote", "add", "origin", remote.to_str().unwrap()],
13621        )
13622        .await
13623        .unwrap();
13624        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13625        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13626        assert_eq!(
13627            resolve_base(&repo, "main", "origin", None)
13628                .await
13629                .unwrap()
13630                .commit,
13631            head
13632        );
13633        #[cfg(unix)]
13634        {
13635            let policy: crate::config::Coordination = toml::from_str(
13636                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13637            ).unwrap();
13638            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13639            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13640            let held = crate::repo_lock::SeatLease::operation_with(
13641                &repo,
13642                Some(&policy),
13643                &crate::repo_lock::WaitPolicy::default(),
13644            )
13645            .await
13646            .unwrap();
13647            assert!(held.is_some());
13648            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13649            tokio::pin!(fetch);
13650            assert!(
13651                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13652                    .await
13653                    .is_err()
13654            );
13655            assert!(
13656                !repo.join(".git/FETCH_HEAD").exists(),
13657                "no fetch before admission"
13658            );
13659            drop(held);
13660            let fetched = fetch.await.unwrap();
13661            assert_eq!(fetched.commit, head);
13662            assert_eq!(fetched.remote, "origin");
13663            assert_eq!(fetched.branch, "main");
13664        }
13665        // A surviving tracking ref is not successful fetch evidence.
13666        git::git(
13667            &repo,
13668            &[
13669                "remote",
13670                "set-url",
13671                "origin",
13672                tmp.path().join("missing").to_str().unwrap(),
13673            ],
13674        )
13675        .await
13676        .unwrap();
13677        let error = resolve_base(&repo, "main", "origin", None)
13678            .await
13679            .unwrap_err()
13680            .to_string();
13681        assert!(error.contains("cannot read origin/main"));
13682        assert!(error.contains("refusing to branch off"));
13683    }
13684
13685    #[test]
13686    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13687        let leaked = vec!["chore(deps): update a crate".to_owned()];
13688        let own = "fix(daemon): apply a chosen action";
13689        assert!(should_retitle("", own, &leaked));
13690        assert!(should_retitle(
13691            &format!("{REVIEW_PROMPT_OPENING} `x`"),
13692            own,
13693            &leaked
13694        ));
13695        assert!(should_retitle(
13696            "chore: land candidate A of run 1",
13697            own,
13698            &leaked
13699        ));
13700        assert!(should_retitle(
13701            "magi: candidate A (uncommitted work)",
13702            own,
13703            &leaked
13704        ));
13705        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13706        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13707        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13708    }
13709
13710    #[test]
13711    fn pr_merge_plan_creates_adopts_or_stops() {
13712        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13713        assert_eq!(
13714            pr_merge_plan(Ok(land::OpenPr::One {
13715                url: "u".into(),
13716                title: "t".into()
13717            })),
13718            PrPlan::Adopt {
13719                url: "u".into(),
13720                title: "t".into()
13721            }
13722        );
13723        let PrPlan::Stop(many) =
13724            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13725        else {
13726            panic!("many must stop");
13727        };
13728        assert!(many.contains('a') && many.contains('b'));
13729        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13730            panic!("a failed lookup must stop");
13731        };
13732        assert!(err.contains("bad token"));
13733    }
13734
13735    use super::*;
13736    use crate::run::GateStatus;
13737    use std::collections::BTreeMap;
13738    use std::time::Duration;
13739
13740    fn conductor() -> AgentSpec {
13741        AgentSpec {
13742            id: "conductor".to_owned(),
13743            kind: crate::config::AgentKind::Command,
13744            model: None,
13745            command: vec!["true".to_owned()],
13746            extra_args: Vec::new(),
13747            env: BTreeMap::new(),
13748            prompt_delivery: None,
13749            max_turn_secs: None,
13750        }
13751    }
13752
13753    fn spec(id: &str) -> AgentSpec {
13754        AgentSpec {
13755            id: id.to_owned(),
13756            kind: crate::config::AgentKind::Command,
13757            model: None,
13758            command: vec!["true".to_owned()],
13759            extra_args: Vec::new(),
13760            env: BTreeMap::new(),
13761            prompt_delivery: None,
13762            max_turn_secs: None,
13763        }
13764    }
13765
13766    fn ids(xs: &[&str]) -> BTreeSet<String> {
13767        xs.iter().map(|s| (*s).to_owned()).collect()
13768    }
13769
13770    #[test]
13771    fn pick_successor_skips_an_agent_another_seat_holds() {
13772        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
13773        let roster = [spec("a"), spec("b"), spec("c")];
13774        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13775        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13776    }
13777
13778    #[test]
13779    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13780        // The other seat started on c but was handed to d; c is free again.
13781        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13782        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13783        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13784        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13785        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13786    }
13787
13788    #[test]
13789    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13790        let roster = [spec("a"), spec("b")];
13791        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13792        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13793        let carried = ids(&["b"]);
13794        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13795        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13796    }
13797
13798    #[test]
13799    fn pick_successor_returns_none_without_any_untried_successor() {
13800        let roster = [spec("a"), spec("b")];
13801        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13802        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13803        let carried = ids(&["a"]);
13804        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13805    }
13806
13807    #[test]
13808    fn pick_successor_rescue_avoids_another_seats_occupant() {
13809        // b is a carried failure and free; a is held by the other seat.
13810        let roster = [spec("a"), spec("b"), spec("c")];
13811        let carried = ids(&["b", "c"]);
13812        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13813        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13814    }
13815
13816    #[test]
13817    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13818        let roster = [spec("a"), spec("b"), spec("c")];
13819        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13820        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13821    }
13822
13823    #[test]
13824    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13825        let roster = [spec("a"), spec("b"), spec("c")];
13826        let failed = ids(&["a", "b", "c"]);
13827        // Rescue looks at the whole roster, once per id, then runs out.
13828        let mut tried = ids(&["b"]);
13829        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13830        assert_eq!(first.id, "a");
13831        tried.insert(first.id.clone());
13832        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13833        assert_eq!(second.id, "c");
13834        tried.insert(second.id.clone());
13835        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13836    }
13837
13838    #[test]
13839    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13840        let roster = [spec("a"), spec("b")];
13841        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13842        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13843    }
13844
13845    #[test]
13846    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13847        let roster = [spec("a"), spec("b"), spec("c")];
13848        let h = SeatHistory {
13849            failed: ids(&["a"]),
13850            last_ok: Some("b".to_owned()),
13851            last_fail: None,
13852        };
13853        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13854        // A last answerer that left the roster, or later failed, is ignored.
13855        let gone = SeatHistory {
13856            last_ok: Some("zzz".to_owned()),
13857            ..h.clone()
13858        };
13859        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13860        let failed = SeatHistory {
13861            failed: ids(&["a", "b"]),
13862            last_ok: Some("b".to_owned()),
13863            last_fail: None,
13864        };
13865        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13866    }
13867
13868    #[test]
13869    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13870        let first = SeatState::new("review-1", "alpha", 7);
13871        let next = handover_seat("review-1", "gamma", 7);
13872        assert_ne!(first.claude_session, next.claude_session);
13873    }
13874
13875    #[test]
13876    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13877        let mut state = state_with_summary("x", "y");
13878        let a = handover_seat("review-1", "beta", state.next_seat_seed());
13879        let b = handover_seat("review-1", "beta", state.next_seat_seed());
13880        assert_ne!(a.claude_session, b.claude_session);
13881    }
13882
13883    #[test]
13884    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13885        let roster = [spec("a"), spec("b")];
13886        let h = SeatHistory {
13887            failed: ids(&["a", "b"]),
13888            ..SeatHistory::default()
13889        };
13890        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13891        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13892        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13893    }
13894
13895    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
13896    // fallback loop depends on to terminate: it must walk forward from the
13897    // seat's own position, never restart at the front of the roster, and it
13898    // must never hand back an id already tried, however many times that id
13899    // happens to appear.
13900
13901    #[test]
13902    fn failure_signature_ignores_numbers_and_paths() {
13903        assert_eq!(
13904            failure_signature("exited with Some(2) and no usable output"),
13905            failure_signature("exited with Some(137) and no usable output")
13906        );
13907        assert_eq!(
13908            failure_signature("process exited with no usable output (exit code 2)"),
13909            failure_signature("process exited with no usable output (exit code 137)")
13910        );
13911        for (old, new) in [
13912            (
13913                "exited with Some(2) and no usable output",
13914                "process exited with no usable output (exit code 2)",
13915            ),
13916            (
13917                "exited with None and no usable output",
13918                "process exited with no usable output (exit code unavailable)",
13919            ),
13920        ] {
13921            let previous = FailClass::Other(failure_signature(old));
13922            let current = FailClass::Other(failure_signature(new));
13923            assert_eq!(previous, current, "resumption retains the failure class");
13924            assert!(!should_hand_over(Some(&previous), &current));
13925        }
13926        assert_eq!(
13927            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13928            failure_signature("cannot open /var/x.txt: denied")
13929        );
13930        assert_ne!(failure_signature("boom"), failure_signature("bang"));
13931    }
13932
13933    #[test]
13934    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13935        let other = FailClass::Other("x".into());
13936        assert!(should_hand_over(None, &FailClass::Quota));
13937        assert!(should_hand_over(Some(&other), &FailClass::Quota));
13938        assert!(should_hand_over(
13939            Some(&FailClass::Timeout),
13940            &FailClass::Timeout
13941        ));
13942        assert!(should_hand_over(None, &other));
13943        assert!(!should_hand_over(Some(&other), &other));
13944        assert!(should_hand_over(
13945            Some(&other),
13946            &FailClass::Other("y".into())
13947        ));
13948        // A quota or timeout in between ends the run of identical failures.
13949        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13950        assert!(should_hand_over(Some(&FailClass::Quota), &other));
13951    }
13952
13953    #[test]
13954    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13955        let a = SeatState::new("judge-1", "alpha", 7);
13956        let b = handover_seat("judge-1", "beta", 7);
13957        assert_ne!(a.claude_session, b.claude_session);
13958        assert_eq!(b.turns, 0);
13959    }
13960
13961    #[test]
13962    fn a_timeout_is_classified_apart_from_other_failures() {
13963        assert_eq!(
13964            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13965            Some(FailClass::Timeout)
13966        );
13967        assert!(matches!(
13968            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13969            Some(FailClass::Other(_))
13970        ));
13971    }
13972
13973    #[test]
13974    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13975        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13976        let tried = BTreeSet::from(["beta".to_owned()]);
13977        // beta sits at index 1; the next candidate is gamma, never alpha —
13978        // which is very likely a different candidate slot's own agent.
13979        let next = next_untried_in_roster(&roster, 1, &tried);
13980        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13981    }
13982
13983    #[test]
13984    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13985        let roster = vec![spec("alpha"), spec("beta")];
13986        let tried = BTreeSet::from(["beta".to_owned()]);
13987        // beta is the roster's last entry: nothing follows it, and alpha —
13988        // earlier in the roster, almost certainly a different candidate
13989        // slot's own agent — must not be reached by wrapping back to it.
13990        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13991    }
13992
13993    #[test]
13994    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13995        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13996        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13997        // beta (index 1) and gamma (index 2, the only entry after it) have
13998        // both been tried; alpha (index 0) never has, but it comes before
13999        // beta's own position, so there is nothing further for this seat.
14000        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14001    }
14002
14003    #[test]
14004    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
14005        let roster = vec![spec("a"), spec("a"), spec("b")];
14006        let tried = BTreeSet::from(["a".to_owned()]);
14007        let next = next_untried_in_roster(&roster, 0, &tried);
14008        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14009    }
14010
14011    #[test]
14012    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
14013        let roster = vec![spec("a"), spec("b")];
14014        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
14015        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
14016    }
14017
14018    #[test]
14019    fn remove_if_empty_only_ever_takes_a_bare_directory() {
14020        let dir = tempfile::tempdir().unwrap();
14021        let bay = dir.path().join("ffff");
14022
14023        // Not there yet: nothing to do, nothing to panic on.
14024        remove_if_empty(&bay);
14025        assert!(!bay.exists());
14026
14027        // Something still inside - the winner's worktree, or a stray file -
14028        // keeps the directory standing.
14029        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14030        remove_if_empty(&bay);
14031        assert!(bay.exists(), "non-empty directory must survive");
14032
14033        // Once the last entry is gone, so is the directory itself.
14034        std::fs::remove_dir(bay.join("cand-A")).unwrap();
14035        remove_if_empty(&bay);
14036        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14037    }
14038
14039    // `round_is_clean` is the exact decision this task fixed: a round with a
14040    // seat that never answered must not read the same as a round every seat
14041    // actually reviewed. These are deterministic and process-free by design —
14042    // the equivalent end-to-end check (a real reviewer timing out under a
14043    // live graph run) is a genuine race against wall-clock contention, and a
14044    // spawn slow enough to blow even a generous budget under a loaded test
14045    // run must not turn this specific regression check flaky.
14046
14047    #[test]
14048    fn a_full_panel_that_found_nothing_is_clean() {
14049        assert!(round_is_clean(
14050            0,
14051            true,
14052            2,
14053            2,
14054            0,
14055            IncompleteReviewPolicy::Block
14056        ));
14057    }
14058
14059    #[test]
14060    fn a_missing_seat_is_never_clean_under_the_default_policy() {
14061        assert!(!round_is_clean(
14062            0,
14063            true,
14064            1,
14065            2,
14066            0,
14067            IncompleteReviewPolicy::Block
14068        ));
14069    }
14070
14071    #[test]
14072    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14073        assert!(!round_is_clean(
14074            1,
14075            true,
14076            1,
14077            2,
14078            0,
14079            IncompleteReviewPolicy::Warn
14080        ));
14081    }
14082
14083    #[test]
14084    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14085        assert!(round_is_clean(
14086            0,
14087            true,
14088            1,
14089            2,
14090            0,
14091            IncompleteReviewPolicy::Warn
14092        ));
14093    }
14094
14095    #[test]
14096    fn a_full_panel_with_an_open_finding_is_not_clean() {
14097        assert!(!round_is_clean(
14098            1,
14099            true,
14100            2,
14101            2,
14102            0,
14103            IncompleteReviewPolicy::Block
14104        ));
14105    }
14106
14107    #[test]
14108    fn a_full_panel_with_a_red_e2e_is_not_clean() {
14109        assert!(!round_is_clean(
14110            0,
14111            false,
14112            2,
14113            2,
14114            0,
14115            IncompleteReviewPolicy::Block
14116        ));
14117    }
14118
14119    // The stall this task closes: under the default `block` policy, a seat
14120    // missing only because it was rate limited must not force a wait for a
14121    // session limit that will not lift by the next round. `round_is_clean`
14122    // is where that quorum carve-out lives; the review loop around it never
14123    // changes what a reviewer's vote or a finding's severity means.
14124
14125    #[test]
14126    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14127        // 1 of 2 answered, and the one missing was quota'd — the exact
14128        // "review-2 rate limited (quota)" shape from the field report.
14129        assert!(round_is_clean(
14130            0,
14131            true,
14132            1,
14133            2,
14134            1,
14135            IncompleteReviewPolicy::Block
14136        ));
14137    }
14138
14139    #[test]
14140    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14141        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
14142        // not a quota loss (`quota_missing` stays 0) — worth another try.
14143        assert!(!round_is_clean(
14144            0,
14145            true,
14146            1,
14147            2,
14148            0,
14149            IncompleteReviewPolicy::Block
14150        ));
14151    }
14152
14153    #[test]
14154    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14155        assert!(!round_is_clean(
14156            1,
14157            true,
14158            1,
14159            2,
14160            1,
14161            IncompleteReviewPolicy::Block
14162        ));
14163        assert!(!round_is_clean(
14164            0,
14165            false,
14166            1,
14167            2,
14168            1,
14169            IncompleteReviewPolicy::Block
14170        ));
14171    }
14172
14173    #[test]
14174    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14175        // Every seat quota'd, nobody answered: there is no panel to decide
14176        // on, so this must fall through to the existing block-and-retry
14177        // fallback rather than call an unreviewed patch clean.
14178        assert!(!round_is_clean(
14179            0,
14180            true,
14181            0,
14182            2,
14183            2,
14184            IncompleteReviewPolicy::Block
14185        ));
14186    }
14187
14188    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14189        CommandOutcome {
14190            command: "test".to_owned(),
14191            code,
14192            output_tail: String::new(),
14193            duration_ms: 0,
14194            resource_blocked,
14195        }
14196    }
14197
14198    #[cfg(unix)]
14199    #[tokio::test]
14200    async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14201        ask_test_home();
14202        let temp = tempfile::tempdir().unwrap();
14203        let mut runner = runner_at(RunStatus::Gating);
14204        let shell = vec!["sh".to_owned(), "-c".to_owned()];
14205        let commands = vec![
14206            "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14207            "touch must-not-run".to_owned(),
14208        ];
14209        let (outcomes, retried, pids) = run_e2e_with_retry(
14210            &mut runner.state,
14211            &shell,
14212            &commands,
14213            temp.path(),
14214            Duration::from_secs(5),
14215            "fixture",
14216        )
14217        .await;
14218        assert_eq!(outcomes.len(), 1);
14219        assert!(outcomes[0].resource_blocked);
14220        assert!(!outcomes[0].build_failed());
14221        assert!(!retried);
14222        assert!(pids.is_empty());
14223        assert!(!temp.path().join("must-not-run").exists());
14224        assert_eq!(
14225            runner.state.resource_block.as_ref().unwrap().kind,
14226            "incremental cache admission"
14227        );
14228    }
14229
14230    #[test]
14231    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14232        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14233        assert!(
14234            !verify_inconclusive(&[outcome(Some(1), false)]),
14235            "an ordinary failure is still evidence about the patch"
14236        );
14237        assert!(verify_inconclusive(&[outcome(None, true)]));
14238        assert!(
14239            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14240            "one inconclusive outcome taints the whole batch"
14241        );
14242        assert!(!verify_inconclusive(&[]));
14243    }
14244
14245    #[tokio::test]
14246    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14247        // Alive for the first two checks, then dead - confirms the loop
14248        // actually re-polls rather than deciding once and sleeping out the
14249        // ceiling regardless.
14250        let calls = std::sync::atomic::AtomicUsize::new(0);
14251        let started = Instant::now();
14252        wait_for_pids_with(
14253            &[123],
14254            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14255            Duration::from_millis(5),
14256            Duration::from_secs(5),
14257        )
14258        .await;
14259        assert!(
14260            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14261            "must keep checking rather than deciding on the first answer"
14262        );
14263        assert!(
14264            started.elapsed() < Duration::from_secs(1),
14265            "must return the moment it is confirmed dead, not wait out the ceiling"
14266        );
14267    }
14268
14269    #[tokio::test]
14270    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14271        let started = Instant::now();
14272        wait_for_pids_with(
14273            &[123],
14274            |_| true, // never reports dead
14275            Duration::from_millis(5),
14276            Duration::from_millis(30),
14277        )
14278        .await;
14279        let elapsed = started.elapsed();
14280        assert!(
14281            elapsed >= Duration::from_millis(30),
14282            "must not give up before its own ceiling: {elapsed:?}"
14283        );
14284        assert!(
14285            elapsed < Duration::from_secs(1),
14286            "must not wait past its own ceiling either: {elapsed:?}"
14287        );
14288    }
14289
14290    #[tokio::test]
14291    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14292        let started = Instant::now();
14293        wait_for_pids_with(
14294            &[],
14295            |_| true,
14296            Duration::from_secs(5),
14297            Duration::from_secs(5),
14298        )
14299        .await;
14300        assert!(
14301            started.elapsed() < Duration::from_millis(200),
14302            "an empty pid list has nothing to confirm"
14303        );
14304    }
14305
14306    // `review_conclusion` is the exact decision the review hand-off task
14307    // fixed: a round budget spent (or a tree that stopped moving) must not
14308    // collapse into `Blocked` regardless of what verification actually
14309    // said. Deterministic and process-free for the same reason the
14310    // `round_is_clean` family above is.
14311    fn review_round(
14312        clean: bool,
14313        blocking: usize,
14314        answered: usize,
14315        expected: usize,
14316        progressed: bool,
14317        e2e_ok: bool,
14318    ) -> ReviewRound {
14319        ReviewRound {
14320            round: 1,
14321            head: "h".to_owned(),
14322            verified_head: None,
14323            verified_at: None,
14324            reviews: Vec::new(),
14325            e2e: vec![CommandOutcome {
14326                command: "test".to_owned(),
14327                code: Some(if e2e_ok { 0 } else { 1 }),
14328                output_tail: String::new(),
14329                duration_ms: 0,
14330                resource_blocked: false,
14331            }],
14332            verify_retried: false,
14333            e2e_deferred: false,
14334            e2e_defer_reason: None,
14335            fix: None,
14336            blocking,
14337            answered,
14338            expected,
14339            clean,
14340            progressed,
14341            vote_split: false,
14342            reconsideration: Vec::new(),
14343            verdict: None,
14344        }
14345    }
14346
14347    #[test]
14348    fn review_conclusion_is_none_when_nothing_has_run() {
14349        assert_eq!(review_conclusion(&[], 3), None);
14350    }
14351
14352    #[test]
14353    fn review_conclusion_is_none_while_rounds_remain() {
14354        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14355        assert_eq!(review_conclusion(&rounds, 3), None);
14356    }
14357
14358    #[test]
14359    fn review_conclusion_is_gating_once_a_round_is_clean() {
14360        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14361        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14362    }
14363
14364    #[test]
14365    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14366        let rounds = vec![
14367            review_round(false, 1, 2, 2, true, true),
14368            review_round(false, 1, 2, 2, true, true),
14369        ];
14370        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14371    }
14372
14373    #[test]
14374    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14375        let rounds = vec![
14376            review_round(false, 1, 2, 2, true, true),
14377            review_round(false, 1, 2, 2, true, false),
14378        ];
14379        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14380    }
14381
14382    #[test]
14383    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14384        // Magi never got a command to run against this round's own head — a
14385        // resource-blocked attempt, not a red one — so this must never
14386        // settle on `Blocked` the way a genuine e2e failure would. `None`
14387        // here is what tells `Runner::review_loop` to retry the check
14388        // itself rather than trust this cheap recomputation with a verdict
14389        // it cannot actually produce.
14390        let mut blocked = review_round(false, 1, 2, 2, true, false);
14391        blocked.e2e[0].resource_blocked = true;
14392        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14393        assert_eq!(review_conclusion(&rounds, 2), None);
14394    }
14395
14396    #[test]
14397    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14398        // Missing input, not a verified tree — never a hand-off candidate.
14399        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14400        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14401    }
14402
14403    #[test]
14404    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14405        let rounds = vec![
14406            review_round(false, 1, 2, 2, false, true),
14407            review_round(false, 1, 2, 2, false, true),
14408        ];
14409        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14410    }
14411
14412    fn secs(n: u64) -> Duration {
14413        Duration::from_secs(n)
14414    }
14415
14416    /// A throwaway repo with one commit on `main`, for tests that need `merge`
14417    /// to make real (and, if it runs at all, real*ly fail*) git calls.
14418    #[tokio::test]
14419    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14420        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14421        if std::env::var_os(CHILD).is_none() {
14422            // Other tests reconcile every run in the process-global home,
14423            // briefly taking owner locks. This ancestry test expects an idle
14424            // predecessor, so give it its own process and pinned test home.
14425            let output = std::process::Command::new(std::env::current_exe().unwrap())
14426                .args([
14427                    "--exact",
14428                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14429                    "--nocapture",
14430                ])
14431                .env(CHILD, "1")
14432                .env_remove("MAGI_WORKER_HOME")
14433                .env_remove("MAGI_WORKER_ID")
14434                .output()
14435                .unwrap();
14436            assert!(
14437                output.status.success()
14438                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14439                "{}\n{}",
14440                String::from_utf8_lossy(&output.stdout),
14441                String::from_utf8_lossy(&output.stderr)
14442            );
14443            return;
14444        }
14445        ask_test_home();
14446        let tmp = tempfile::tempdir().unwrap();
14447        let repo = tmp.path().join("repo");
14448        std::fs::create_dir_all(&repo).unwrap();
14449        init_repo(&repo);
14450        let g = |args: &[&str]| {
14451            let out = std::process::Command::new("git")
14452                .args(args)
14453                .current_dir(&repo)
14454                .quiet()
14455                .output()
14456                .unwrap();
14457            assert!(
14458                out.status.success(),
14459                "{}",
14460                String::from_utf8_lossy(&out.stderr)
14461            );
14462        };
14463        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14464        g(&["checkout", "-b", "historical"]);
14465        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14466        g(&["add", "."]);
14467        g(&["commit", "-m", "old alternative"]);
14468        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14469        g(&["checkout", "-b", "magi/task-selected", &base]);
14470        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14471        g(&["add", "."]);
14472        g(&["commit", "-m", "measurement infrastructure"]);
14473        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14474        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14475        g(&["add", "."]);
14476        g(&["commit", "-m", "dependent implementation"]);
14477        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14478        g(&["checkout", "main"]);
14479        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14480        g(&["add", "."]);
14481        g(&["commit", "-m", "unrelated base change"]);
14482        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14483        let origin = tmp.path().join("origin.git");
14484        g(&[
14485            "clone",
14486            "--bare",
14487            repo.to_str().unwrap(),
14488            origin.to_str().unwrap(),
14489        ]);
14490        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14491        g(&["fetch", "origin"]);
14492        let selection = crate::pr_recovery::Selection {
14493            url: "https://example.test/project/pull/1".into(),
14494            branch: "magi/task-selected".into(),
14495            head: head.clone(),
14496            source_head: None,
14497            source_run: None,
14498            source_proof: None,
14499            disposition: Some(crate::pr_recovery::Disposition::Review),
14500            instruction: "preserve measured behavior".into(),
14501            approved_at: jiff::Timestamp::now(),
14502        };
14503        let mut template = runner_at(RunStatus::Reviewing);
14504        template.state.repo = repo.clone();
14505        template.state.base_commit = current.clone();
14506        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14507        template.state.origin = Some(crate::run::Origin::queue("selected"));
14508        template.state.pr_head = Some(selection.branch.clone());
14509        template.state.pr_recovery = Some(selection.clone());
14510        let mut runner =
14511            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14512                .await
14513                .unwrap();
14514        let wt = runner.state.winner().unwrap().worktree.clone();
14515        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14516        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14517        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14518        assert!(runner.state.seeds.is_empty());
14519        assert_eq!(runner.state.instruction, "preserve measured behavior");
14520        runner.sync_to_base().await.unwrap();
14521        assert_eq!(
14522            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14523            "prerequisite"
14524        );
14525        assert_eq!(
14526            std::fs::read_to_string(wt.join("implementation")).unwrap(),
14527            "measured implementation"
14528        );
14529        assert_eq!(
14530            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14531            "current main change"
14532        );
14533        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14534        assert_eq!(
14535            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14536            head
14537        );
14538
14539        // A failed selected-source setup stays inspectable, never resumable as a new competition.
14540        let mut partial = runner_at(RunStatus::Reviewing);
14541        partial.state.repo = repo.clone();
14542        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14543        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14544            run: Some("exit 1".into()),
14545            ..Default::default()
14546        }];
14547        let selection = runner.state.pr_recovery.clone().unwrap();
14548        partial.state.pr_recovery = Some(selection.clone());
14549        partial.state.pr_head = Some(selection.branch.clone());
14550        partial.state.origin = Some(crate::run::Origin::queue("selected"));
14551        let id = partial.state.id.clone();
14552        let error = match Runner::open_selected(
14553            &repo,
14554            partial.state,
14555            partial.roles,
14556            1,
14557            runner.state.base_commit.clone(),
14558            selection,
14559        )
14560        .await
14561        {
14562            Ok(_) => panic!("setup must fail"),
14563            Err(e) => e,
14564        };
14565        assert_eq!(
14566            error
14567                .downcast_ref::<crate::run::StartupFailure>()
14568                .unwrap()
14569                .run,
14570            id
14571        );
14572        let failed = RunState::load(&id).unwrap();
14573        assert_eq!(failed.status, RunStatus::Failed);
14574        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14575        assert!(Runner::resume(&id).is_err());
14576        runner.state.status = RunStatus::Ready;
14577        runner.state.driver_exited = true;
14578        runner.state.save().unwrap();
14579        let mut retry = runner_at(RunStatus::Reviewing);
14580        retry.state.repo = repo.clone();
14581        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14582        let mut reselected = runner.state.pr_recovery.clone().unwrap();
14583        reselected.approved_at = jiff::Timestamp::now();
14584        retry.state.origin = Some(crate::run::Origin::queue("selected"));
14585        retry.state.pr_head = Some(reselected.branch.clone());
14586        retry.state.pr_recovery = Some(reselected.clone());
14587        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14588            .await
14589            .unwrap();
14590        let restarted = Runner::open_selected(
14591            &repo,
14592            retry.state,
14593            retry.roles,
14594            1,
14595            runner.state.base_commit.clone(),
14596            reselected,
14597        )
14598        .await
14599        .unwrap();
14600        assert_eq!(
14601            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14602                .await
14603                .unwrap(),
14604            head
14605        );
14606        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14607        assert_eq!(
14608            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14609            head
14610        );
14611    }
14612
14613    #[tokio::test]
14614    #[cfg(unix)]
14615    async fn fallback_and_interrupted_restoration_require_coordination() {
14616        use std::os::fd::AsRawFd as _;
14617        ask_test_home();
14618        let tmp = tempfile::tempdir().unwrap();
14619        let mut runner = registered_runner(tmp.path()).await;
14620        runner.state.config.graph.timeout_resource_wait = 1;
14621        let winner = runner.state.candidates[0].clone();
14622        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14623        let leftover = winner.worktree.join("leftover");
14624        std::fs::write(&leftover, "untrusted edits").unwrap();
14625        let next = tmp.path().join("next-fixer");
14626        let mut failed = spec("first");
14627        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14628        let mut successor = spec("second");
14629        successor.command = vec![
14630            "sh".into(),
14631            "-c".into(),
14632            format!("touch '{}'; echo done", next.display()),
14633        ];
14634        let attempts = vec![
14635            (failed.clone(), "first".into()),
14636            (successor.clone(), "second".into()),
14637        ];
14638        let lock_path = runner.state.repo.join(".git/updater.lock");
14639        let external = std::fs::File::create(&lock_path).unwrap();
14640        assert_eq!(
14641            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14642            0
14643        );
14644        let artifacts = tmp.path().join("artifacts");
14645        let build = |spec, seat| SeatJob {
14646            spec,
14647            seat,
14648            cwd: winner.worktree.clone(),
14649            prompt: "fixture".into(),
14650            timeout: Duration::from_secs(10),
14651            allow_write: true,
14652            sessions: false,
14653            artifacts: artifacts.clone(),
14654            stem: "fix".into(),
14655            handover: None,
14656        };
14657        let (_, _, out) = runner
14658            .ask_fixer_among(
14659                attempts.clone(),
14660                "graft",
14661                None,
14662                Some((&winner.worktree, &head)),
14663                build,
14664            )
14665            .await;
14666        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14667        assert!(leftover.exists(), "contention must prevent clean");
14668        assert!(!next.exists());
14669        assert!(runner.state.handovers.is_empty());
14670        // Early interrupted-repair rollback uses the same protected route.
14671        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14672            from_run: "earlier".into(),
14673            generation: 1,
14674            mode: crate::recovery::Mode::VerifyRepair,
14675            reasons: Vec::new(),
14676            reviewed_head: None,
14677            rounds: Vec::new(),
14678            open_findings: Vec::new(),
14679            evidence: None,
14680            criteria_inherited: true,
14681            repair: crate::recovery::Repair::Started {
14682                before: head.clone(),
14683            },
14684        });
14685        assert!(
14686            runner
14687                .recovery_repair(&winner)
14688                .await
14689                .unwrap_err()
14690                .is::<crate::repo_lock::ResourceBlocked>()
14691        );
14692        assert!(matches!(
14693            runner.state.recovery.as_ref().unwrap().repair,
14694            crate::recovery::Repair::Started { .. }
14695        ));
14696        assert!(leftover.exists());
14697        drop(external);
14698        runner.recovery_repair(&winner).await.unwrap();
14699        assert!(!leftover.exists());
14700        assert!(matches!(
14701            runner.state.recovery.as_ref().unwrap().repair,
14702            crate::recovery::Repair::Done { .. }
14703        ));
14704        // A hard lock-open failure also leaves HEAD and untracked work intact.
14705        std::fs::remove_file(&lock_path).unwrap();
14706        std::fs::create_dir(&lock_path).unwrap();
14707        std::fs::write(&leftover, "untrusted edits").unwrap();
14708        let (_, _, out) = runner
14709            .ask_fixer_among(
14710                attempts,
14711                "graft",
14712                None,
14713                Some((&winner.worktree, &head)),
14714                build,
14715            )
14716            .await;
14717        assert!(matches!(out, AgentOutcome::Failed(_)));
14718        assert!(leftover.exists());
14719        assert!(!next.exists());
14720        assert_eq!(
14721            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14722            head
14723        );
14724    }
14725
14726    #[cfg(unix)]
14727    async fn registered_runner(root: &Path) -> Runner {
14728        repository_runner(root, crate::config::test_coordination()).await
14729    }
14730
14731    async fn repository_runner(
14732        root: &Path,
14733        coordination: Option<crate::config::Coordination>,
14734    ) -> Runner {
14735        ask_test_home();
14736        let repo = root.join("repo");
14737        std::fs::create_dir_all(&repo).unwrap();
14738        init_repo(&repo);
14739        if coordination.is_some() {
14740            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14741        }
14742        let mut runner = runner_at(RunStatus::Prep);
14743        runner.state.repo = repo.clone();
14744        runner.state.config.coordination = coordination;
14745        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14746        runner.state.config.graph.worktree_root = Some(root.join("wt"));
14747        runner.state.config.graph.advise = false;
14748        runner.roles.implementers = vec![spec("implement")];
14749        runner.roles.judges = vec![spec("judge")];
14750        runner.prep().await.unwrap();
14751        runner
14752    }
14753
14754    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14755
14756    #[tokio::test]
14757    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14758        let tmp = tempfile::tempdir().unwrap();
14759        let mut runner = repository_runner(tmp.path(), None).await;
14760        let prompts = runner.state.config.prompts.clone();
14761        let run = runner.state.id.clone();
14762        let ctx = WaveCtx {
14763            carry_seats: false,
14764            pauses: None,
14765            run: &run,
14766            node: "advise",
14767            prompts: &prompts,
14768            cache: None,
14769            round: None,
14770        };
14771        let jobs: Vec<_> = ["alpha", "beta"]
14772            .into_iter()
14773            .map(|id| SeatJob {
14774                spec: scripted(id, AUTH_OUTAGE),
14775                seat: SeatState::new(id, id, 7),
14776                prompt: "propose".into(),
14777                cwd: runner.state.candidates[0].worktree.clone(),
14778                timeout: Duration::from_secs(10),
14779                allow_write: false,
14780                sessions: false,
14781                artifacts: agent::artifacts_dir(&runner.state.dir()),
14782                stem: id.into(),
14783                handover: None,
14784            })
14785            .collect();
14786        let mut losses = Vec::new();
14787        let results = ask_wave_with(
14788            jobs.clone(),
14789            runner.sem.clone(),
14790            2,
14791            &[],
14792            &ctx,
14793            &mut losses,
14794            &mut runner.state,
14795            &|_: &str| Ok(()),
14796        )
14797        .await;
14798        assert!(
14799            results
14800                .iter()
14801                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14802        );
14803        assert_eq!(runner.state.quota.len(), 2);
14804        assert!(
14805            runner
14806                .state
14807                .quota
14808                .iter()
14809                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14810        );
14811        assert!(!runner.state.advise_attempted);
14812        runner.stop_for_resource().unwrap();
14813        assert_eq!(runner.state.status, RunStatus::Blocked);
14814        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14815        assert_eq!(
14816            runner.state.resource_block.as_ref().unwrap().kind,
14817            "agent authentication"
14818        );
14819        // A machine loss followed by an ordinary failure is not evidence
14820        // that the whole node was unavailable, and must not earn a refund.
14821        runner.state.resource_block = None;
14822        runner.state.quota.clear();
14823        let mut mixed = jobs;
14824        mixed[1].spec.command = vec!["false".into()];
14825        let results = ask_wave_with(
14826            mixed,
14827            runner.sem.clone(),
14828            0,
14829            &[],
14830            &ctx,
14831            &mut losses,
14832            &mut runner.state,
14833            &|_: &str| Ok(()),
14834        )
14835        .await;
14836        assert!(results.iter().all(|(_, res, _)| res.is_err()));
14837        assert!(runner.state.resource_block.is_none());
14838        assert_eq!(losses.len(), 1);
14839    }
14840
14841    #[tokio::test]
14842    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14843        let tmp = tempfile::tempdir().unwrap();
14844        let mut runner = repository_runner(tmp.path(), None).await;
14845        let implementer = scripted("implement", AUTH_OUTAGE);
14846        runner.roles.implementers = vec![implementer.clone()];
14847        runner.state.config.agents = vec![implementer];
14848        runner.implement().await.unwrap();
14849        assert_eq!(runner.state.status, RunStatus::Blocked);
14850        assert_eq!(runner.state.quota.len(), 1);
14851        assert_eq!(
14852            runner.state.quota[0].cause,
14853            agent::QuotaCause::Authentication
14854        );
14855        let candidate = &runner.state.candidates[0];
14856        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14857        assert_eq!(runner.state.seats["impl-A"].turns, 1);
14858    }
14859
14860    #[tokio::test]
14861    #[cfg(unix)]
14862    async fn parked_implementer_persists_completed_work() {
14863        for registered in [false, true] {
14864            let tmp = tempfile::tempdir().unwrap();
14865            let mut runner = registered_runner(tmp.path()).await;
14866            if !registered {
14867                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14868            }
14869            let marker = tmp.path().join("started");
14870            let release = tmp.path().join("release");
14871            let mut implementer = spec("implement");
14872            implementer.command = vec![
14873                "sh".into(),
14874                "-c".into(),
14875                format!(
14876                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14877                    marker.display(),
14878                    release.display()
14879                ),
14880            ];
14881            runner.roles.implementers = vec![implementer.clone()];
14882            runner.state.config.agents = vec![implementer];
14883            let pause = runner.pause.clone();
14884            let (result, ()) = tokio::join!(runner.implement(), async {
14885                tokio::time::timeout(Duration::from_secs(10), async {
14886                    while !marker.exists() {
14887                        tokio::time::sleep(Duration::from_millis(10)).await;
14888                    }
14889                })
14890                .await
14891                .unwrap();
14892                pause.park();
14893                std::fs::write(&release, "go").unwrap();
14894            });
14895            result.unwrap();
14896            let candidate = &runner.state.candidates[0];
14897            assert!(candidate.commits > 0, "{candidate:?}");
14898            assert!(candidate.summary.contains("saved-result"));
14899            assert_eq!(candidate.files, 1);
14900            let before = candidate.commits;
14901            runner.implement().await.unwrap();
14902            assert_eq!(runner.state.candidates[0].commits, before);
14903        }
14904    }
14905
14906    #[tokio::test]
14907    #[cfg(unix)]
14908    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14909        let tmp = tempfile::tempdir().unwrap();
14910        let mut runner = registered_runner(tmp.path()).await;
14911        let winner = runner.state.candidates[0].clone();
14912        let cache = tmp.path().join("cache");
14913        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14914        runner.state.config.graph.timeout_resource_wait = 0;
14915        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14916        let crate::cache::AcquireOutcome::Acquired(held) =
14917            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14918        else {
14919            panic!("cache must be free")
14920        };
14921        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14922            from_run: "earlier".into(),
14923            generation: 1,
14924            mode: crate::recovery::Mode::VerifyRepair,
14925            reasons: Vec::new(),
14926            reviewed_head: None,
14927            rounds: Vec::new(),
14928            open_findings: Vec::new(),
14929            evidence: None,
14930            criteria_inherited: true,
14931            repair: crate::recovery::Repair::NotStarted,
14932        });
14933        runner.recovery_repair(&winner).await.unwrap();
14934        assert!(matches!(
14935            runner.state.recovery.as_ref().unwrap().repair,
14936            crate::recovery::Repair::WaitingForRestore { .. }
14937        ));
14938        drop(held);
14939        runner.state.resource_block = None;
14940        runner.state.config.graph.timeout_resource_wait = 1;
14941        runner.recovery_repair(&winner).await.unwrap();
14942        assert!(matches!(
14943            runner.state.recovery.as_ref().unwrap().repair,
14944            crate::recovery::Repair::Done { .. }
14945        ));
14946        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14947    }
14948
14949    #[tokio::test]
14950    #[cfg(unix)]
14951    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14952        use std::os::fd::AsRawFd as _;
14953        for graft in [false, true] {
14954            let tmp = tempfile::tempdir().unwrap();
14955            let mut runner = registered_runner(tmp.path()).await;
14956            runner.state.config.graph.timeout_resource_wait = 1;
14957            let winner = runner.state.candidates[0].clone();
14958            let marker = tmp.path().join("fixer-started");
14959            let mut fixer = spec("fixer");
14960            fixer.command = vec![
14961                "sh".into(),
14962                "-c".into(),
14963                format!("echo started >> '{}'; echo done", marker.display()),
14964            ];
14965            runner.roles.fixer = Some(vec![fixer]);
14966            if graft {
14967                let mut loser = winner.clone();
14968                loser.label = 'B';
14969                loser.index = 1;
14970                runner.state.candidates.push(loser);
14971                runner.state.config.graph.graft_runner_up = true;
14972                runner.state.tally = Some(Tally {
14973                    first_choice: BTreeMap::new(),
14974                    borda: BTreeMap::new(),
14975                    winner: winner.label,
14976                    rankings: 1,
14977                    unanimous_initial: true,
14978                    deliberated: false,
14979                    changed_votes: 0,
14980                    unanimous_final: true,
14981                    tie_break: None,
14982                    judges: 1,
14983                    present: 1,
14984                    quorum: 1,
14985                    met_quorum: true,
14986                    uncontested: None,
14987                });
14988                runner.state.judgements.push(crate::run::Judgement {
14989                    judge: 1,
14990                    seat: "judge-1".into(),
14991                    agent: "judge".into(),
14992                    ranking: vec![winner.label, 'B'],
14993                    reasons: BTreeMap::new(),
14994                    confidence: None,
14995                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14996                    order: vec![0, 1],
14997                    failed: None,
14998                    duration_ms: 0,
14999                });
15000            } else {
15001                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15002                    from_run: "earlier".into(),
15003                    generation: 1,
15004                    mode: crate::recovery::Mode::VerifyRepair,
15005                    reasons: Vec::new(),
15006                    reviewed_head: None,
15007                    rounds: Vec::new(),
15008                    open_findings: Vec::new(),
15009                    evidence: None,
15010                    criteria_inherited: true,
15011                    repair: crate::recovery::Repair::NotStarted,
15012                });
15013            }
15014            let reader = crate::repo_lock::SeatLease::for_checkout(
15015                &winner.worktree,
15016                false,
15017                crate::config::test_coordination().as_ref(),
15018            )
15019            .await
15020            .unwrap();
15021            let pause = runner.pause.clone();
15022            let (result, ()) = tokio::join!(
15023                async {
15024                    if graft {
15025                        runner.graft().await
15026                    } else {
15027                        runner.recovery_repair(&winner).await
15028                    }
15029                },
15030                async {
15031                    tokio::time::sleep(Duration::from_millis(100)).await;
15032                    pause.park();
15033                }
15034            );
15035            result.unwrap();
15036            assert!(!marker.exists(), "a blocked seat must not spawn");
15037            // Exercise the persisted state, as a binary replacement/resume does.
15038            runner.state = RunState::load(&runner.state.id).unwrap();
15039            if graft {
15040                assert_eq!(
15041                    runner.state.graft.as_ref().unwrap().outcome,
15042                    GraftOutcome::WaitingForRestore
15043                );
15044            } else {
15045                assert!(matches!(
15046                    runner.state.recovery.as_ref().unwrap().repair,
15047                    crate::recovery::Repair::WaitingForRestore { .. }
15048                ));
15049            }
15050            drop(reader);
15051            runner.pause = Pause::new();
15052            runner.state.resource_block = None;
15053            // Deferred restoration must still respect an external lock owner.
15054            let external =
15055                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15056            assert_eq!(
15057                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15058                0
15059            );
15060            let leftover = winner.worktree.join("leftover");
15061            std::fs::write(&leftover, "earlier fallback edits").unwrap();
15062            let result = if graft {
15063                runner.graft().await
15064            } else {
15065                runner.recovery_repair(&winner).await
15066            };
15067            assert!(
15068                result
15069                    .unwrap_err()
15070                    .is::<crate::repo_lock::ResourceBlocked>()
15071            );
15072            assert!(leftover.exists());
15073            assert!(!marker.exists());
15074            drop(external);
15075            runner.state.resource_block = None;
15076            let result = if graft {
15077                runner.graft().await
15078            } else {
15079                runner.recovery_repair(&winner).await
15080            };
15081            result.unwrap();
15082            assert!(!leftover.exists(), "restore must precede retry");
15083            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15084            if graft {
15085                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15086            } else {
15087                assert!(matches!(
15088                    runner.state.recovery.as_ref().unwrap().repair,
15089                    crate::recovery::Repair::Done { .. }
15090                ));
15091            }
15092        }
15093    }
15094
15095    #[tokio::test]
15096    #[cfg(unix)]
15097    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15098        use std::os::fd::AsRawFd as _;
15099        let tmp = tempfile::tempdir().unwrap();
15100        let mut runner = registered_runner(tmp.path()).await;
15101        runner.state.config.graph.timeout_resource_wait = 1;
15102        let winner = runner.state.candidates[0].clone();
15103        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15104        runner.state.tally = Some(Tally {
15105            first_choice: BTreeMap::new(),
15106            borda: BTreeMap::new(),
15107            winner: winner.label,
15108            rankings: 0,
15109            unanimous_initial: false,
15110            deliberated: false,
15111            changed_votes: 0,
15112            unanimous_final: false,
15113            tie_break: None,
15114            judges: 0,
15115            present: 0,
15116            quorum: 0,
15117            met_quorum: true,
15118            uncontested: Some("fixture".into()),
15119        });
15120        runner.state.graft = Some(GraftRecord {
15121            pre: pre.clone(),
15122            post: None,
15123            lines: None,
15124            limit: None,
15125            agent: None,
15126            outcome: GraftOutcome::Running,
15127            requested: Vec::new(),
15128            adoption: None,
15129            adoption_error: None,
15130        });
15131        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15132        git::commit_all(&winner.worktree, "unfinished graft")
15133            .await
15134            .unwrap();
15135        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15136        let leftover = winner.worktree.join("leftover");
15137        std::fs::write(&leftover, "untrusted edits").unwrap();
15138        let path = runner.state.repo.join(".git/updater.lock");
15139        let external = std::fs::File::create(&path).unwrap();
15140        assert_eq!(
15141            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15142            0
15143        );
15144        assert!(
15145            runner
15146                .graft()
15147                .await
15148                .unwrap_err()
15149                .is::<crate::repo_lock::ResourceBlocked>()
15150        );
15151        assert!(matches!(
15152            runner.state.graft.as_ref().unwrap().outcome,
15153            GraftOutcome::Running
15154        ));
15155        assert_eq!(
15156            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15157            unfinished
15158        );
15159        assert!(leftover.exists());
15160        drop(external);
15161        std::fs::remove_file(&path).unwrap();
15162        std::fs::create_dir(&path).unwrap();
15163        assert!(runner.graft().await.is_err());
15164        assert!(matches!(
15165            runner.state.graft.as_ref().unwrap().outcome,
15166            GraftOutcome::Running
15167        ));
15168        assert_eq!(
15169            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15170            unfinished
15171        );
15172        assert!(leftover.exists());
15173        std::fs::remove_dir(&path).unwrap();
15174        runner.graft().await.unwrap();
15175        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15176        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15177        assert!(!leftover.exists());
15178    }
15179
15180    #[tokio::test]
15181    #[cfg(unix)]
15182    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15183        use std::os::fd::AsRawFd as _;
15184        let tmp = tempfile::tempdir().unwrap();
15185        let mut runner = registered_runner(tmp.path()).await;
15186        runner.state.prep_complete = Some(false);
15187        runner.state.config.graph.timeout_resource_wait = 1;
15188        runner.state.save().unwrap();
15189        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15190        assert_eq!(
15191            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15192            0
15193        );
15194        runner.execute().await.unwrap();
15195        let disk = RunState::load(&runner.state.id).unwrap();
15196        assert!(disk.driver_exited);
15197        assert_eq!(disk.driver_pid, None);
15198        assert_eq!(runner.state.status, RunStatus::Blocked);
15199        assert_eq!(
15200            runner.state.resource_block.as_ref().unwrap().kind,
15201            "repository coordination"
15202        );
15203        assert!(runner.state.prep_failure.is_none());
15204        assert!(runner.state.seats.is_empty());
15205        runner.pause.park();
15206        // Include headroom for Git subprocesses under parallel test load;
15207        // the repository contention deadline itself remains one second.
15208        tokio::time::timeout(Duration::from_secs(10), runner.execute())
15209            .await
15210            .unwrap()
15211            .unwrap();
15212        assert!(runner.state.parked);
15213        assert!(runner.state.prep_failure.is_none());
15214        let disk = RunState::load(&runner.state.id).unwrap();
15215        assert!(disk.driver_exited);
15216        assert_eq!(disk.driver_pid, None);
15217    }
15218
15219    #[tokio::test]
15220    #[cfg(unix)]
15221    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15222        use std::os::fd::AsRawFd as _;
15223        let tmp = tempfile::tempdir().unwrap();
15224        let mut runner = registered_runner(tmp.path()).await;
15225        runner.state.prep_complete = Some(false);
15226        runner.state.config.graph.timeout_resource_wait = 30;
15227        runner.state.save().unwrap();
15228        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15229        assert_eq!(
15230            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15231            0
15232        );
15233        assert!(
15234            tokio::time::timeout(Duration::from_millis(100), runner.execute())
15235                .await
15236                .is_err()
15237        );
15238        let disk = RunState::load(&runner.state.id).unwrap();
15239        assert!(disk.driver_exited);
15240        assert_eq!(disk.driver_pid, None);
15241        assert!(disk.status.resumable());
15242        assert!(runner.state.driver_exited);
15243        assert_eq!(runner.state.driver_pid, None);
15244        runner.state.save().unwrap();
15245        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15246    }
15247
15248    #[tokio::test]
15249    #[cfg(unix)]
15250    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15251        let tmp = tempfile::tempdir().unwrap();
15252        let mut runner = registered_runner(tmp.path()).await;
15253        let winner = runner.state.candidates[0].clone();
15254        let cache = tmp.path().join("cache");
15255        let started = tmp.path().join("pre-gate-started");
15256        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15257        runner.state.config.verify.gate =
15258            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15259        let home = crate::run::home();
15260        let owner =
15261            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15262        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15263            crate::cache::AcquireOutcome::Acquired(g) => g,
15264            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15265        };
15266        let mut gate = Box::pin(runner.run_gate(&winner));
15267        assert!(
15268            tokio::time::timeout(Duration::from_millis(150), &mut gate)
15269                .await
15270                .is_err()
15271        );
15272        assert!(!started.exists(), "pre-gate must wait for the cache too");
15273        // Simulate the wave that already owns the cache entering invoke.
15274        // It must obtain the editor reservation and finish without waiting
15275        // for the gate that is queued behind its cache.
15276        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15277        let editor = crate::repo_lock::SeatLease::checkout_with(
15278            &winner.worktree,
15279            true,
15280            crate::config::test_coordination().as_ref(),
15281            &wait,
15282        )
15283        .await
15284        .unwrap();
15285        drop(editor);
15286        guard.release();
15287        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15288            .await
15289            .unwrap()
15290            .unwrap();
15291        drop(gate);
15292        assert!(outcomes.iter().all(CommandOutcome::ok));
15293        assert!(started.exists());
15294    }
15295
15296    #[tokio::test]
15297    #[cfg(unix)]
15298    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15299        use std::os::fd::AsRawFd as _;
15300
15301        ask_test_home();
15302        let tmp = tempfile::tempdir().unwrap();
15303        let repo = tmp.path().join("repo");
15304        std::fs::create_dir_all(&repo).unwrap();
15305        init_repo(&repo);
15306        let common = repo.join(".git");
15307        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15308        let mut runner = runner_at(RunStatus::Prep);
15309        runner.state.repo = repo.clone();
15310        runner.state.config.coordination = crate::config::test_coordination();
15311        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15312        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15313        runner.state.config.graph.advise = false;
15314        runner.roles.implementers = vec![spec("implement")];
15315        runner.roles.judges = vec![spec("judge")];
15316        runner.prep().await.unwrap();
15317        let winner = runner.state.candidates[0].clone();
15318        // Emulate edits already left by a completed seat. The fixture CLI
15319        // only reports completion; it performs no repository operations.
15320        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15321        let completed = tmp.path().join("seat-completed");
15322        runner.roles.implementers[0].command = vec![
15323            "sh".into(),
15324            "-c".into(),
15325            format!("touch '{}'; echo completed", completed.display()),
15326        ];
15327        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15328        assert_eq!(
15329            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15330            0
15331        );
15332        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15333        let mut implement = Box::pin(runner.implement());
15334        tokio::select! {
15335            result = &mut implement => panic!("rescue must wait: {result:?}"),
15336            result = tokio::time::timeout(Duration::from_secs(10), async {
15337                while !completed.exists() {
15338                    tokio::time::sleep(Duration::from_millis(20)).await;
15339                }
15340            }) => result.unwrap(),
15341        }
15342        assert!(
15343            tokio::time::timeout(Duration::from_millis(200), &mut implement)
15344                .await
15345                .is_err()
15346        );
15347        assert_eq!(
15348            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15349            before
15350        );
15351        assert_eq!(
15352            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15353            0
15354        );
15355        tokio::time::timeout(Duration::from_secs(10), &mut implement)
15356            .await
15357            .unwrap()
15358            .unwrap();
15359        drop(implement);
15360        assert_ne!(
15361            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15362            before
15363        );
15364        assert!(git::is_clean(&winner.worktree).await.unwrap());
15365
15366        let started = tmp.path().join("pre-gate-started");
15367        let proceed = tmp.path().join("pre-gate-proceed");
15368        let checked = tmp.path().join("gate-ran");
15369        runner.state.config.verify.pre_gate = vec![format!(
15370            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15371            started.display(),
15372            proceed.display()
15373        )];
15374        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15375        assert_eq!(
15376            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15377            0
15378        );
15379        let mut gate = Box::pin(runner.run_gate(&winner));
15380        assert!(
15381            tokio::time::timeout(Duration::from_millis(200), &mut gate)
15382                .await
15383                .is_err()
15384        );
15385        assert!(!started.exists());
15386        assert!(!checked.exists());
15387        assert!(git::is_clean(&winner.worktree).await.unwrap());
15388        assert_eq!(
15389            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15390            0
15391        );
15392        tokio::select! {
15393            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15394            result = tokio::time::timeout(Duration::from_secs(10), async {
15395                while !started.exists() {
15396                    tokio::time::sleep(Duration::from_millis(20)).await;
15397                }
15398            }) => result.unwrap(),
15399        }
15400        assert_eq!(
15401            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15402            -1,
15403            "mechanical edits must hold the real lock"
15404        );
15405        std::fs::write(proceed, "ready").unwrap();
15406        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15407            .await
15408            .unwrap()
15409            .unwrap();
15410        drop(gate);
15411        assert!(outcomes.iter().all(CommandOutcome::ok));
15412        assert!(checked.exists());
15413        assert!(runner.state.pre_gate_commit.is_some());
15414        assert!(git::is_clean(&winner.worktree).await.unwrap());
15415        assert_eq!(
15416            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15417            0,
15418            "gate completion must release coordination"
15419        );
15420        drop(outsider);
15421        std::fs::remove_file(common.join("updater.lock")).unwrap();
15422        std::fs::create_dir(common.join("updater.lock")).unwrap();
15423        std::fs::remove_file(&started).unwrap();
15424        std::fs::remove_file(&checked).unwrap();
15425        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15426        let error = runner.run_gate(&winner).await.unwrap_err();
15427        assert!(
15428            error
15429                .to_string()
15430                .contains("acquire mandatory repository lock")
15431        );
15432        assert!(
15433            !started.exists(),
15434            "failed acquisition must prevent pre-gate"
15435        );
15436        assert!(!checked.exists(), "failed acquisition must prevent gate");
15437        assert!(git::is_clean(&winner.worktree).await.unwrap());
15438        assert_eq!(
15439            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15440            head
15441        );
15442    }
15443
15444    #[tokio::test]
15445    #[cfg(unix)]
15446    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15447        use std::os::fd::AsRawFd as _;
15448        use std::time::Duration;
15449
15450        ask_test_home();
15451        let tmp = tempfile::tempdir().unwrap();
15452        let repo = tmp.path().join("repo");
15453        std::fs::create_dir_all(&repo).unwrap();
15454        init_repo(&repo);
15455        let common = repo.join(".git");
15456        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15457        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15458        assert_eq!(
15459            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15460            0
15461        );
15462        let mut runner = runner_at(RunStatus::Prep);
15463        runner.state.repo = repo.clone();
15464        runner.state.config.coordination = crate::config::test_coordination();
15465        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15466        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15467        runner.state.config.graph.advise = true;
15468        runner.state.config.graph.advisors = 1;
15469        runner.roles.implementers = vec![spec("implement")];
15470        runner.roles.judges = vec![spec("judge")];
15471        let root = runner.state.worktree_root();
15472        let started = tmp.path().join("setup-started");
15473        let proceed = tmp.path().join("setup-proceed");
15474        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15475            run: Some(format!(
15476                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15477                started.display(),
15478                proceed.display()
15479            )),
15480            ..Default::default()
15481        }];
15482        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15483        let config = git::git(&repo, &["config", "--local", "--list"])
15484            .await
15485            .unwrap();
15486        let mut prep = Box::pin(runner.prep());
15487        assert!(
15488            tokio::time::timeout(Duration::from_millis(200), &mut prep)
15489                .await
15490                .is_err()
15491        );
15492        assert!(
15493            !root.exists(),
15494            "contention must prevent even checkout creation"
15495        );
15496        assert!(!started.exists(), "contention must prevent setup");
15497        assert_eq!(
15498            git::git(&repo, &["branch", "--list"]).await.unwrap(),
15499            branches
15500        );
15501        assert_eq!(
15502            git::git(&repo, &["config", "--local", "--list"])
15503                .await
15504                .unwrap(),
15505            config
15506        );
15507        drop(lock);
15508        tokio::select! {
15509            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15510            result = tokio::time::timeout(Duration::from_secs(10), async {
15511                while !started.exists() {
15512                    tokio::time::sleep(Duration::from_millis(20)).await;
15513                }
15514            }) => result.unwrap(),
15515        }
15516        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15517        assert_eq!(
15518            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15519            -1
15520        );
15521        assert!(
15522            tokio::time::timeout(
15523                Duration::from_millis(100),
15524                crate::repo_lock::SeatLease::for_checkout(
15525                    &repo,
15526                    true,
15527                    crate::config::test_coordination().as_ref()
15528                )
15529            )
15530            .await
15531            .is_err(),
15532            "preparation must also exclude local editors"
15533        );
15534        std::fs::write(proceed, "ready").unwrap();
15535        tokio::time::timeout(Duration::from_secs(10), &mut prep)
15536            .await
15537            .unwrap()
15538            .unwrap();
15539        drop(prep);
15540        assert_eq!(runner.state.prep_complete, Some(true));
15541        assert!(root.join("judge-1").exists());
15542        assert!(root.join("advisor-1").exists());
15543        assert_eq!(
15544            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15545            0
15546        );
15547    }
15548
15549    #[tokio::test]
15550    #[cfg(unix)]
15551    async fn preparation_lock_failure_prevents_repository_changes() {
15552        ask_test_home();
15553        let tmp = tempfile::tempdir().unwrap();
15554        let repo = tmp.path().join("repo");
15555        std::fs::create_dir_all(&repo).unwrap();
15556        init_repo(&repo);
15557        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15558        // A directory at the required lock path makes opening the real lock
15559        // fail, rather than merely exercising an invalid registration.
15560        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15561        let mut runner = runner_at(RunStatus::Prep);
15562        runner.state.repo = repo.clone();
15563        runner.state.config.coordination = crate::config::test_coordination();
15564        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15565        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15566        runner.roles.implementers = vec![spec("implement")];
15567        let error = runner.prep().await.unwrap_err();
15568        assert!(
15569            error
15570                .to_string()
15571                .contains("acquire mandatory repository lock")
15572        );
15573        assert!(!runner.state.worktree_root().exists());
15574        assert!(runner.state.candidates.is_empty());
15575        assert!(!repo.join(".git/config.worktree").exists());
15576        assert_eq!(
15577            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15578            "* main"
15579        );
15580    }
15581
15582    #[tokio::test]
15583    async fn preparation_records_creation_intent_before_git_can_fail() {
15584        ask_test_home();
15585        let tmp = tempfile::tempdir().unwrap();
15586        let repo = tmp.path().join("repo");
15587        std::fs::create_dir_all(&repo).unwrap();
15588        init_repo(&repo);
15589        let mut runner = runner_at(RunStatus::Prep);
15590        runner.state.repo = repo.clone();
15591        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15592        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15593        runner.state.config.blind.commit_msg_hook = false;
15594        runner.roles.implementers = vec![spec("one")];
15595        let label = blind::assign_labels(1, runner.state.seed)[0];
15596        let branch = runner.state.branch_for(label);
15597        // A conflicting branch makes Git refuse creation before a candidate
15598        // can be registered. The durable intent must still identify it.
15599        git::git(&repo, &["branch", &branch]).await.unwrap();
15600        assert!(runner.prep().await.is_err());
15601        let saved = RunState::load(&runner.state.id).unwrap();
15602        assert!(saved.candidates.is_empty());
15603        let event = saved
15604            .events
15605            .iter()
15606            .find(|e| e.node == "prep-intent")
15607            .unwrap();
15608        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15609        assert_eq!(intent["branch"], branch);
15610        assert_eq!(intent["start"], runner.state.base_commit);
15611        assert_eq!(
15612            intent["worktree"],
15613            runner
15614                .state
15615                .worktree_root()
15616                .join(format!("cand-{label}"))
15617                .to_string_lossy()
15618                .as_ref()
15619        );
15620        assert_eq!(intent["picks"], serde_json::json!([]));
15621    }
15622
15623    #[tokio::test]
15624    async fn invalid_pinned_sources_create_no_partial_worktrees() {
15625        ask_test_home();
15626        let tmp = tempfile::tempdir().unwrap();
15627        let repo = tmp.path().join("repo");
15628        std::fs::create_dir_all(&repo).unwrap();
15629        init_repo(&repo);
15630        let mut runner = runner_at(RunStatus::Prep);
15631        runner.state.repo = repo.clone();
15632        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15633        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15634        runner.state.sources.pick_commits.push(refs::Input {
15635            token: "missing".into(),
15636            sha: "0".repeat(40),
15637            provenance: refs::Provenance::ExplicitCli,
15638        });
15639        assert!(runner.prep().await.is_err());
15640        assert!(runner.state.candidates.is_empty());
15641        assert!(!runner.state.worktree_root().exists());
15642    }
15643
15644    #[tokio::test]
15645    async fn prose_and_legacy_inference_never_import_into_candidates() {
15646        ask_test_home();
15647        let tmp = tempfile::tempdir().unwrap();
15648        let repo = tmp.path().join("repo");
15649        std::fs::create_dir_all(&repo).unwrap();
15650        init_repo(&repo);
15651        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15652        git::git(&repo, &["checkout", "-b", "magi/history/A"])
15653            .await
15654            .unwrap();
15655        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15656        git::git(&repo, &["add", "."]).await.unwrap();
15657        git::git(&repo, &["commit", "-m", "historical implementation"])
15658            .await
15659            .unwrap();
15660        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15661        git::git(&repo, &["checkout", "main"]).await.unwrap();
15662        for instruction in [
15663            format!("Example: magi/history/A at {source}"),
15664            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15665            format!("Never import magi/history/A or {source}"),
15666            format!("Conversation answer: I mentioned {source} only as background"),
15667        ] {
15668            let mut runner = runner_at(RunStatus::Prep);
15669            runner.state.repo = repo.clone();
15670            runner.state.base_commit = base.clone();
15671            runner.state.instruction = instruction.clone();
15672            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15673            runner.state.config.blind.commit_msg_hook = false;
15674            runner.state.seeds = vec![refs::Seed {
15675                token: "magi/history/A".into(),
15676                sha: source.clone(),
15677                kind: refs::SeedKind::Unmerged,
15678                branch: true,
15679                detail: String::new(),
15680            }];
15681            runner.roles.implementers = vec![spec("one")];
15682            assert_eq!(seeded_instruction(&runner.state), instruction);
15683            // Model an old record with inferred seeds but no structured input field.
15684            let mut legacy = serde_json::to_value(&runner.state).unwrap();
15685            legacy.as_object_mut().unwrap().remove("sources");
15686            runner.state = serde_json::from_value(legacy).unwrap();
15687            assert_eq!(runner.state.sources, refs::Sources::default());
15688            runner.state.save().unwrap();
15689            runner.state = RunState::load(&runner.state.id).unwrap();
15690            runner.prep().await.unwrap();
15691            for candidate in &runner.state.candidates {
15692                assert_eq!(
15693                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15694                    base
15695                );
15696                assert!(!candidate.worktree.join("unwanted").exists());
15697            }
15698        }
15699    }
15700
15701    #[tokio::test]
15702    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15703        ask_test_home();
15704        let tmp = tempfile::tempdir().unwrap();
15705        let repo = tmp.path().join("repo");
15706        std::fs::create_dir_all(&repo).unwrap();
15707        init_repo(&repo);
15708        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15709        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15710        std::fs::write(repo.join("referenced"), "required commit").unwrap();
15711        git::git(&repo, &["add", "."]).await.unwrap();
15712        git::git(&repo, &["commit", "-m", "required seed"])
15713            .await
15714            .unwrap();
15715        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15716        git::git(&repo, &["checkout", "main"]).await.unwrap();
15717        let mut runner = runner_at(RunStatus::Prep);
15718        runner.state.repo = repo.clone();
15719        runner.state.base_commit = base.clone();
15720        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15721        runner.state.config.graph.advise = false;
15722        runner.state.config.blind.commit_msg_hook = false;
15723        runner.state.sources =
15724            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15725                .await
15726                .unwrap();
15727        runner.roles.implementers = vec![spec("one"), spec("two")];
15728        runner.roles.judges = vec![spec("judge")];
15729        let label = blind::assign_labels(2, runner.state.seed)[0];
15730        let branch = runner.state.branch_for(label);
15731        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15732        git::worktree_add_branch(&repo, &worktree, &branch, &base)
15733            .await
15734            .unwrap();
15735        runner.state.candidates.push(Candidate {
15736            index: 0,
15737            label,
15738            agent: "one".into(),
15739            branch,
15740            worktree,
15741            summary: String::new(),
15742            stat: String::new(),
15743            files: 0,
15744            commits: 0,
15745            empty: false,
15746            failed: None,
15747            verified_noop: None,
15748            duration_ms: 0,
15749            folded: false,
15750        });
15751        runner.state.save().unwrap();
15752        // Reload the checkpoint immediately before its first cherry-pick.
15753        runner.state = RunState::load(&runner.state.id).unwrap();
15754        assert_eq!(runner.state.prep_complete, Some(false));
15755        runner.prep().await.unwrap();
15756        assert_eq!(runner.state.prep_complete, Some(true));
15757        assert_eq!(runner.state.candidates.len(), 2);
15758        for candidate in &runner.state.candidates {
15759            assert_eq!(
15760                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15761                git::tree_of(&repo, &seed).await.unwrap()
15762            );
15763            assert_eq!(
15764                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15765                "required commit"
15766            );
15767        }
15768        assert!(runner.state.worktree_root().join("judge-1").exists());
15769        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15770        runner.prep().await.unwrap();
15771        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15772
15773        runner.state.prep_complete = Some(false);
15774        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15775            run: Some(format!(
15776                "test -f '{}'",
15777                tmp.path().join("dependency-ready").display()
15778            )),
15779            ..Default::default()
15780        }];
15781        assert!(runner.prep().await.is_err());
15782        assert!(runner.state.prep_failure.is_some());
15783        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15784        runner.state = RunState::load(&runner.state.id).unwrap();
15785        assert!(
15786            runner.prep().await.is_err(),
15787            "correcting the cause alone must not auto-retry"
15788        );
15789        let failed_at = runner.state.events.last().unwrap().at;
15790        assert!(
15791            runner.retry_preparation(failed_at).is_err(),
15792            "old approvals cannot unlock the hold"
15793        );
15794        let mut task = crate::queue::Task::new(
15795            "retry".into(),
15796            "retry setup".into(),
15797            repo.clone(),
15798            crate::queue::Source::Human,
15799        );
15800        task.start(runner.state.id.clone());
15801        task.hold_machine(Some("setup failed".into()));
15802        task.release_operator("cli", None);
15803        runner
15804            .retry_preparation(task.resumption.as_ref().unwrap().at)
15805            .unwrap();
15806        runner.prep().await.unwrap();
15807        assert!(runner.state.prep_failure.is_none());
15808        assert_eq!(runner.state.prep_complete, Some(true));
15809        assert_eq!(runner.state.candidates.len(), 2);
15810        assert!(runner.state.worktree_root().join("judge-1").exists());
15811        for candidate in &runner.state.candidates {
15812            assert_eq!(
15813                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15814                git::tree_of(&repo, &seed).await.unwrap()
15815            );
15816        }
15817    }
15818
15819    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15820        let mut heads = Vec::new();
15821        for candidate in candidates {
15822            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15823        }
15824        heads
15825    }
15826
15827    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15828        ask_test_home();
15829        let tmp = tempfile::tempdir().unwrap();
15830        let repo = tmp.path().join("repo");
15831        std::fs::create_dir(&repo).unwrap();
15832        init_repo(&repo);
15833        let mut runner = runner_at(RunStatus::Reviewing);
15834        runner.state.repo = repo.clone();
15835        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15836        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15837        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15838            let branch = format!("candidate-{label}");
15839            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15840            git::worktree_add_branch(&repo, &worktree, &branch, "main")
15841                .await
15842                .unwrap();
15843            if label == 'B' {
15844                std::fs::write(worktree.join("change"), "winner").unwrap();
15845                git::rescue_commit(&worktree, "candidate change")
15846                    .await
15847                    .unwrap();
15848            }
15849            runner.state.candidates.push(Candidate {
15850                index,
15851                label,
15852                agent: "alpha".into(),
15853                branch,
15854                worktree,
15855                summary: "completed reply".into(),
15856                stat: String::new(),
15857                files: usize::from(label == 'B'),
15858                commits: usize::from(label == 'B'),
15859                empty: label == 'A',
15860                failed: None,
15861                verified_noop: None,
15862                duration_ms: 1,
15863                folded: false,
15864            });
15865        }
15866        runner.state.judge_skipped = true;
15867        runner.state.tally = Some(Tally {
15868            first_choice: BTreeMap::new(),
15869            borda: BTreeMap::new(),
15870            winner: 'B',
15871            rankings: 0,
15872            unanimous_initial: false,
15873            deliberated: false,
15874            changed_votes: 0,
15875            unanimous_final: false,
15876            tie_break: None,
15877            judges: 0,
15878            present: 0,
15879            quorum: 0,
15880            met_quorum: true,
15881            uncontested: Some("only B produced a change".into()),
15882        });
15883        runner.state.save().unwrap();
15884        (tmp, runner)
15885    }
15886
15887    #[tokio::test]
15888    #[cfg(unix)]
15889    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15890        for (solo, failed_checkpoint) in
15891            [(true, false), (false, false), (true, true), (false, true)]
15892        {
15893            let (_tmp, mut runner) = preservation_resume_fixture().await;
15894            if solo {
15895                runner.state.candidates.retain(|c| c.label == 'A');
15896            }
15897            runner.state.candidates[0].empty = false;
15898            runner.state.candidates[0].summary.clear();
15899            runner.state.tally = None;
15900            runner.state.judge_skipped = false;
15901            runner.state.status = RunStatus::Implementing;
15902            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15903            let mut implementer = spec("alpha");
15904            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15905            runner.roles.implementers = vec![implementer];
15906            let patch = runner.state.dir().join("artifacts/cand-A.patch");
15907            if failed_checkpoint {
15908                std::fs::create_dir_all(&patch).unwrap();
15909            }
15910            runner.state.save().unwrap();
15911            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15912            drop(pressure);
15913            runner.state = RunState::load(&runner.state.id).unwrap();
15914            assert_eq!(runner.state.status, RunStatus::Blocked);
15915            assert!(runner.state.interrupted_implementations.contains(&'A'));
15916            assert_eq!(runner.state.candidates[0].commits, 0);
15917            assert!(!runner.state.candidates[0].empty);
15918            assert!(runner.state.candidates[0].failed.is_none());
15919            if failed_checkpoint {
15920                assert!(runner.state.candidate_preservation.contains_key(&'A'));
15921                std::fs::remove_dir(patch).unwrap();
15922            }
15923            runner.state.resource_block = None;
15924            runner.state.config.disk.min_free_bytes = 0;
15925            runner.roles.implementers[0].command = vec![
15926                "sh".into(),
15927                "-c".into(),
15928                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15929            ];
15930            runner.implement().await.unwrap();
15931            assert!(runner.state.interrupted_implementations.is_empty());
15932            assert!(runner.state.candidate_preservation.is_empty());
15933            assert_eq!(runner.state.candidates[0].commits, 1);
15934            assert!(!runner.state.candidates[0].empty);
15935            assert!(
15936                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15937                    .unwrap()
15938                    .contains("retried empty interrupted seat")
15939            );
15940            if !solo {
15941                assert_eq!(runner.state.candidates[1].commits, 1);
15942            }
15943        }
15944    }
15945
15946    #[tokio::test]
15947    #[cfg(unix)]
15948    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15949        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15950            let (_tmp, mut runner) = preservation_resume_fixture().await;
15951            runner.state.candidates.retain(|c| c.label == 'A');
15952            runner.state.candidates[0].empty = false;
15953            runner.state.candidates[0].summary.clear();
15954            runner.state.tally = None;
15955            runner.state.judge_skipped = false;
15956            runner.state.status = RunStatus::Implementing;
15957            // Force only the real post-spawn watchdog, without failing the
15958            // separate admission check. Existing edits predate its measurement.
15959            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15960            let wt = runner.state.candidates[0].worktree.clone();
15961            std::fs::write(
15962                wt.join("interrupted-source"),
15963                "retain implementation work\n",
15964            )
15965            .unwrap();
15966            if committed {
15967                git::rescue_commit(&wt, "implementation checkpoint")
15968                    .await
15969                    .unwrap();
15970            }
15971            let lock = git::git(
15972                &wt,
15973                &[
15974                    "rev-parse",
15975                    "--path-format=absolute",
15976                    "--git-path",
15977                    "index.lock",
15978                ],
15979            )
15980            .await
15981            .unwrap();
15982            if block_rescue {
15983                std::fs::write(&lock, "fixture-owned lock").unwrap();
15984            }
15985            let mut implementer = spec("alpha");
15986            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15987            runner.roles.implementers = vec![implementer];
15988            runner.state.save().unwrap();
15989            let result = runner.implement().await;
15990            drop(pressure);
15991            assert_eq!(result.is_err(), block_rescue);
15992            assert_eq!(
15993                runner.state.resource_block.as_ref().unwrap().kind,
15994                "disk pressure"
15995            );
15996            assert!(
15997                runner
15998                    .state
15999                    .events
16000                    .iter()
16001                    .any(|e| e.message.contains("agent execution interrupted"))
16002            );
16003            assert!(
16004                runner
16005                    .state
16006                    .dir()
16007                    .join("artifacts/candidate-A-recovery/staged.patch")
16008                    .exists()
16009            );
16010            if block_rescue {
16011                let saved = RunState::load(&runner.state.id).unwrap();
16012                assert!(saved.candidate_preservation.contains_key(&'A'));
16013                assert!(!saved.candidates[0].empty);
16014                assert_eq!(
16015                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
16016                    "retain implementation work\n"
16017                );
16018                std::fs::remove_file(lock).unwrap();
16019                runner.state = saved;
16020            } else {
16021                assert_eq!(runner.state.status, RunStatus::Blocked);
16022                assert_eq!(runner.state.candidates[0].commits, 1);
16023            }
16024            // Recovery must save the candidate before deciding whether another
16025            // implementation is needed. No implementer is available on retry.
16026            runner.roles.implementers.clear();
16027            runner.state.config.disk.min_free_bytes = 0;
16028            runner.state.resource_block = None;
16029            runner.implement().await.unwrap();
16030            assert!(runner.state.candidate_preservation.is_empty());
16031            assert_eq!(runner.state.candidates[0].commits, 1);
16032            assert!(!runner.state.candidates[0].empty);
16033            assert!(git::is_clean(&wt).await.unwrap());
16034            assert!(
16035                crate::run::read_artifact(&runner.state, "cand-A.patch")
16036                    .unwrap()
16037                    .contains("retain implementation work")
16038            );
16039        }
16040    }
16041
16042    #[tokio::test]
16043    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16044        let (_tmp, mut runner) = preservation_resume_fixture().await;
16045        // Schema 15 already confirmed the unfolded no-change result.
16046        runner.implement().await.unwrap();
16047        assert!(runner.state.candidate_preservation.is_empty());
16048        assert!(
16049            !runner
16050                .state
16051                .dir()
16052                .join("artifacts/candidate-A-recovery")
16053                .exists()
16054        );
16055        runner.fold_losers().await.unwrap();
16056        assert!(!runner.state.candidates[0].worktree.exists());
16057        for schema in [14, crate::run::SCHEMA] {
16058            runner.state.schema = schema;
16059            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16060            runner.state.save().unwrap();
16061            runner.state = RunState::load(&runner.state.id).unwrap();
16062            runner.implement().await.unwrap();
16063            assert!(runner.state.candidate_preservation.is_empty());
16064            assert!(runner.state.candidates[0].folded);
16065            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16066        }
16067    }
16068
16069    #[tokio::test]
16070    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16071        let (_tmp, mut runner) = preservation_resume_fixture().await;
16072        runner.state.candidates.retain(|c| c.label == 'A');
16073        runner.state.tally = None;
16074        runner.state.judge_skipped = false;
16075        let candidate = &mut runner.state.candidates[0];
16076        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16077        candidate.failed = Some("deployment validation unavailable".into());
16078        std::fs::write(
16079            candidate.worktree.join("repair"),
16080            "future launcher policy\n",
16081        )
16082        .unwrap();
16083        runner
16084            .state
16085            .candidate_preservation
16086            .insert('A', "awaiting persistence".into());
16087        runner.persist_candidates().await.unwrap();
16088        let candidate = &runner.state.candidates[0];
16089        assert!(!candidate.empty);
16090        assert_eq!(candidate.commits, 1);
16091        assert_eq!(candidate.files, 1);
16092        assert!(candidate.failed.is_none());
16093        assert!(candidate.summary.contains("EPERM"));
16094        assert!(
16095            crate::run::read_artifact(&runner.state, "cand-A.patch")
16096                .unwrap()
16097                .contains("future launcher policy")
16098        );
16099        assert!(runner.state.candidate_preservation.is_empty());
16100    }
16101
16102    #[tokio::test]
16103    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16104        let (_tmp, mut runner) = preservation_resume_fixture().await;
16105        let wt = runner.state.candidates[0].worktree.clone();
16106        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16107        runner.state.schema = 14;
16108        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16109        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16110        let tally = serde_json::to_value(&runner.state.tally).unwrap();
16111        for _ in 0..2 {
16112            assert!(runner.implement().await.is_err());
16113            assert_eq!(runner.state.status, RunStatus::Blocked);
16114            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16115            assert!(
16116                runner.state.candidates[0].empty,
16117                "old verdict must remain unusable"
16118            );
16119            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16120            assert!(runner.state.judge_skipped);
16121            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16122            assert!(
16123                crate::run::read_artifact(&runner.state, "cand-A.patch")
16124                    .unwrap()
16125                    .contains("owned edits")
16126            );
16127            assert!(wt.exists());
16128            assert_eq!(
16129                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16130                    .await
16131                    .unwrap(),
16132                1
16133            );
16134            runner.state = RunState::load(&runner.state.id).unwrap();
16135        }
16136    }
16137
16138    #[tokio::test]
16139    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16140        let (_tmp, mut runner) = preservation_resume_fixture().await;
16141        let wt = runner.state.candidates[0].worktree.clone();
16142        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16143        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16144        let index_lock = PathBuf::from(
16145            git::git(
16146                &wt,
16147                &[
16148                    "rev-parse",
16149                    "--path-format=absolute",
16150                    "--git-path",
16151                    "index.lock",
16152                ],
16153            )
16154            .await
16155            .unwrap(),
16156        );
16157        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16158        runner.state.schema = 14;
16159        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16160        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16161        assert!(runner.implement().await.is_err());
16162        assert!(runner.after_implement().is_err());
16163        assert!(runner.fold_losers().await.is_err());
16164        assert_eq!(
16165            std::fs::read_to_string(wt.join("recovered")).unwrap(),
16166            "failed seat edits"
16167        );
16168        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16169        runner.state = RunState::load(&runner.state.id).unwrap();
16170        std::fs::remove_file(index_lock).unwrap();
16171        // Recovery finds changes that the old uncontested result never judged.
16172        assert!(runner.implement().await.is_err());
16173        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16174        assert!(runner.state.candidates[0].empty);
16175        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16176        assert!(
16177            crate::run::read_artifact(&runner.state, "cand-A.patch")
16178                .unwrap()
16179                .contains("failed seat edits")
16180        );
16181        assert!(wt.exists());
16182    }
16183
16184    #[tokio::test]
16185    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16186        let (_tmp, mut runner) = preservation_resume_fixture().await;
16187        let repo = runner.state.repo.clone();
16188        let common = git::git(
16189            &repo,
16190            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16191        )
16192        .await
16193        .unwrap();
16194        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16195        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16196        assert!(runner.fold_losers().await.is_err());
16197        assert!(!runner.state.candidates[0].worktree.exists());
16198        assert!(!runner.state.candidates[0].folded);
16199        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16200        runner.state = RunState::load(&runner.state.id).unwrap();
16201        std::fs::remove_file(branch_lock).unwrap();
16202        let judge = runner.state.worktree_root().join("judge-1");
16203        git::worktree_add_detached(&repo, &judge, "main")
16204            .await
16205            .unwrap();
16206        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16207        runner.roles.judges = vec![spec("judge")];
16208        assert!(runner.fold_losers().await.is_err());
16209        runner.state = RunState::load(&runner.state.id).unwrap();
16210        assert!(
16211            runner.state.candidates[0].folded,
16212            "candidate progress saved before judge failure"
16213        );
16214        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16215        assert!(judge.join("uncommitted").exists());
16216        std::fs::remove_file(judge.join("uncommitted")).unwrap();
16217        runner.fold_losers().await.unwrap();
16218        assert!(!judge.exists());
16219        runner.fold_losers().await.unwrap();
16220    }
16221
16222    #[tokio::test]
16223    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16224        let (_tmp, mut runner) = preservation_resume_fixture().await;
16225        runner.state.candidates.retain(|c| c.label == 'B');
16226        runner.state.tally = None;
16227        runner.state.judge_skipped = false;
16228        runner.state.prep_complete = Some(true);
16229        runner.state.config.graph.advise = false;
16230        runner.state.config.graph.selected_implementer = Some("alpha".into());
16231        let wt = runner.state.candidates[0].worktree.clone();
16232        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16233        runner
16234            .state
16235            .candidate_preservation
16236            .insert('B', "awaiting persistence".into());
16237        let index_lock = git::git(
16238            &wt,
16239            &[
16240                "rev-parse",
16241                "--path-format=absolute",
16242                "--git-path",
16243                "index.lock",
16244            ],
16245        )
16246        .await
16247        .unwrap();
16248        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16249        runner.state.save().unwrap();
16250        assert!(runner.execute_graph().await.is_err());
16251        assert_eq!(runner.state.status, RunStatus::Blocked);
16252        assert!(runner.state.candidate_preservation.contains_key(&'B'));
16253        std::fs::remove_file(index_lock).unwrap();
16254        // Park at the next graph boundary: this tests the real selected-seat
16255        // hold guard without dispatching reviewers or publishing anything.
16256        for _ in 0..2 {
16257            runner.state = RunState::load(&runner.state.id).unwrap();
16258            runner.pause.park();
16259            runner.execute_graph().await.unwrap();
16260            assert_eq!(runner.state.status, RunStatus::Implementing);
16261            assert!(runner.state.candidate_preservation.is_empty());
16262            assert_eq!(runner.state.candidates[0].commits, 2);
16263            assert!(runner.state.tally.is_none());
16264        }
16265        runner.implement().await.unwrap();
16266        assert_eq!(runner.state.status, RunStatus::Judging);
16267        // A genuine selected-implementer hold must still require release.
16268        runner.state.status = RunStatus::Blocked;
16269        runner.state.candidates[0].failed = Some("implementation failed".into());
16270        runner.state.save().unwrap();
16271        let error = runner.execute_graph().await.unwrap_err();
16272        assert!(
16273            error
16274                .to_string()
16275                .contains("selected implementation is held")
16276        );
16277    }
16278
16279    #[tokio::test]
16280    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16281        let (_tmp, mut runner) = preservation_resume_fixture().await;
16282        let repo = runner.state.repo.clone();
16283        let candidate = runner.state.candidates[0].clone();
16284        // Remove only the disposable fixture directory, leaving Git's linked
16285        // worktree registration and branch intact, as in a vanished checkout.
16286        let registered = git::worktree_holding(&repo, &candidate.branch)
16287            .await
16288            .unwrap()
16289            .unwrap();
16290        assert_eq!(
16291            registered.canonicalize().unwrap(),
16292            candidate.worktree.canonicalize().unwrap()
16293        );
16294        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16295        assert_eq!(
16296            git::worktree_holding(&repo, &candidate.branch)
16297                .await
16298                .unwrap(),
16299            Some(registered)
16300        );
16301        assert!(
16302            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16303            "the stale registration initially keeps the branch checked out"
16304        );
16305        runner.fold_losers().await.unwrap();
16306        assert!(runner.state.candidates[0].folded);
16307        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16308        assert_eq!(
16309            git::worktree_holding(&repo, &candidate.branch)
16310                .await
16311                .unwrap(),
16312            None
16313        );
16314        runner.state = RunState::load(&runner.state.id).unwrap();
16315        runner.fold_losers().await.unwrap();
16316    }
16317
16318    #[tokio::test]
16319    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16320        let (_tmp, mut runner) = preservation_resume_fixture().await;
16321        let repo = runner.state.repo.clone();
16322        let candidate = runner.state.candidates[0].clone();
16323        let winner = runner.state.candidates[1].clone();
16324        let registered = git::worktree_holding(&repo, &candidate.branch)
16325            .await
16326            .unwrap()
16327            .unwrap();
16328        assert_eq!(
16329            registered.canonicalize().unwrap(),
16330            candidate.worktree.canonicalize().unwrap()
16331        );
16332        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16333        assert_eq!(
16334            git::worktree_holding(&repo, &candidate.branch)
16335                .await
16336                .unwrap(),
16337            Some(registered)
16338        );
16339        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16340
16341        for _ in 0..2 {
16342            fold_run(&mut runner.state, false, &crate::run::home())
16343                .await
16344                .unwrap();
16345            runner.state = RunState::load(&runner.state.id).unwrap();
16346            assert!(runner.state.candidates[0].folded);
16347            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16348            assert_eq!(
16349                git::worktree_holding(&repo, &candidate.branch)
16350                    .await
16351                    .unwrap(),
16352                None
16353            );
16354            assert!(!runner.state.candidates[1].folded);
16355            assert!(winner.worktree.exists());
16356            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16357        }
16358    }
16359
16360    #[tokio::test]
16361    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16362        ask_test_home();
16363        let tmp = tempfile::tempdir().unwrap();
16364        let repo = tmp.path().join("repo");
16365        std::fs::create_dir(&repo).unwrap();
16366        init_repo(&repo);
16367        let mut runner = runner_at(RunStatus::Implementing);
16368        runner.state.repo = repo.clone();
16369        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16370        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16371        for (index, label) in ['A', 'B'].into_iter().enumerate() {
16372            let branch = format!("candidate-{label}");
16373            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16374            git::worktree_add_branch(&repo, &wt, &branch, "main")
16375                .await
16376                .unwrap();
16377            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16378            git::git(&wt, &["add", "README.md"]).await.unwrap();
16379            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16380            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16381            runner.state.candidates.push(Candidate {
16382                index,
16383                label,
16384                agent: "alpha".to_owned(),
16385                branch,
16386                worktree: wt,
16387                summary: "completed implementation".to_owned(),
16388                stat: String::new(),
16389                files: 0,
16390                commits: 0,
16391                empty: false,
16392                failed: None,
16393                verified_noop: None,
16394                duration_ms: 42,
16395                folded: false,
16396            });
16397            runner
16398                .state
16399                .candidate_preservation
16400                .insert(label, "awaiting persistence".to_owned());
16401        }
16402        let wt = runner.state.candidates[0].worktree.clone();
16403        let index_lock = git::git(
16404            &wt,
16405            &[
16406                "rev-parse",
16407                "--path-format=absolute",
16408                "--git-path",
16409                "index.lock",
16410            ],
16411        )
16412        .await
16413        .unwrap();
16414        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16415        runner.state.save().unwrap();
16416        assert!(runner.persist_candidates().await.is_err());
16417        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16418        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16419        assert_eq!(
16420            runner.state.candidates[1].commits, 1,
16421            "other results are still collected"
16422        );
16423        assert!(
16424            !runner.state.candidates[0].empty,
16425            "failed rescue is not no change"
16426        );
16427        assert_eq!(
16428            runner.state.candidates[0].summary,
16429            "completed implementation"
16430        );
16431        assert!(runner.after_implement().is_err());
16432        let home = tmp.path().join("fold-home");
16433        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16434        assert!(wt.join("new-file").exists());
16435        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16436        assert!(recovery.join("staged.patch").exists());
16437        assert!(recovery.join("unstaged.patch").exists());
16438        assert_eq!(
16439            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16440            "untracked edits\n"
16441        );
16442        runner.state.begin_pass();
16443        runner.state.end_pass_parked(ParkCause::Upgrade {
16444            from: "1.0".into(),
16445            to: Some("1.1".into()),
16446        });
16447        runner.state.save().unwrap();
16448        runner.state = RunState::load(&runner.state.id).unwrap();
16449        assert!(matches!(
16450            runner.state.passes.last().unwrap().end,
16451            Some(crate::run::PassEnd::Parked {
16452                cause: ParkCause::Upgrade { .. }
16453            })
16454        ));
16455        std::fs::remove_file(index_lock).unwrap();
16456        // No implementer is configured: a persistence retry cannot dispatch a seat.
16457        runner.implement().await.unwrap();
16458        assert!(runner.state.candidate_preservation.is_empty());
16459        assert!(
16460            runner
16461                .state
16462                .candidates
16463                .iter()
16464                .all(|c| c.commits == 1 && !c.empty)
16465        );
16466        assert!(git::is_clean(&wt).await.unwrap());
16467        assert!(
16468            crate::run::read_artifact(&runner.state, "cand-A.patch")
16469                .unwrap()
16470                .contains("later tracked edits")
16471        );
16472    }
16473
16474    #[tokio::test]
16475    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16476        ask_test_home();
16477        let tmp = tempfile::tempdir().unwrap();
16478        let repo = tmp.path().join("repo");
16479        std::fs::create_dir(&repo).unwrap();
16480        init_repo(&repo);
16481        let wt = tmp.path().join("candidate");
16482        git::worktree_add_branch(&repo, &wt, "candidate", "main")
16483            .await
16484            .unwrap();
16485        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16486        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16487        assert!(wt.join("uncommitted").exists());
16488        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16489        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16490        let mut runner = runner_at(RunStatus::Implementing);
16491        runner.state.repo = repo.clone();
16492        runner.state.base_commit = "missing-base".to_owned();
16493        runner.state.candidates.push(Candidate {
16494            index: 0,
16495            label: 'A',
16496            agent: "alpha".into(),
16497            branch: "candidate".into(),
16498            worktree: wt.clone(),
16499            summary: "completed".into(),
16500            stat: String::new(),
16501            files: 0,
16502            commits: 0,
16503            empty: false,
16504            failed: None,
16505            verified_noop: None,
16506            duration_ms: 1,
16507            folded: false,
16508        });
16509        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16510        assert!(
16511            fold_run(&mut runner.state, true, &tmp.path().join("home"))
16512                .await
16513                .is_err()
16514        );
16515        assert!(!runner.state.candidates[0].folded);
16516        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16517        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16518        runner
16519            .state
16520            .candidate_preservation
16521            .insert('A', "pending".into());
16522        assert!(runner.persist_candidates().await.is_err());
16523        assert!(!runner.state.candidates[0].empty);
16524        assert!(!runner.state.candidates[0].folded);
16525        assert!(wt.exists());
16526        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16527        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16528        std::fs::create_dir_all(&patch_path).unwrap();
16529        assert!(
16530            runner.persist_candidates().await.is_err(),
16531            "artifact failure stays pending"
16532        );
16533        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16534        assert!(!runner.state.candidates[0].empty);
16535        std::fs::remove_dir(patch_path).unwrap();
16536        runner.persist_candidates().await.unwrap();
16537        assert!(
16538            runner.state.candidates[0].empty,
16539            "only successful inspection confirms no change"
16540        );
16541    }
16542
16543    fn init_repo(dir: &Path) {
16544        let run = |args: &[&str]| {
16545            let out = std::process::Command::new("git")
16546                .args(args)
16547                .current_dir(dir)
16548                .quiet()
16549                .output()
16550                .expect("spawn git");
16551            assert!(
16552                out.status.success(),
16553                "git {args:?} failed: {}",
16554                String::from_utf8_lossy(&out.stderr)
16555            );
16556        };
16557        run(&["init", "-b", "main"]);
16558        run(&["config", "user.name", "magi test"]);
16559        run(&["config", "user.email", "magi@example.com"]);
16560        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16561        run(&["add", "-A"]);
16562        run(&["commit", "-m", "init"]);
16563    }
16564
16565    // `settle_questions` is what closes the ghost the phone showed: a run's
16566    // seat asked something, the run then ended, and nothing was left to
16567    // abandon the question it left `open`. `HOME` is a process-wide
16568    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
16569    // first time it runs in the binary — every test below still reaches the
16570    // same directory whichever call won, and each gets its own run id from
16571    // `RunState::new`, so they never collide there.
16572    fn ask_test_home() {
16573        crate::run::pin_test_home();
16574    }
16575
16576    #[test]
16577    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16578        ask_test_home();
16579        let mut runner = runner_at(RunStatus::Prep);
16580        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16581        // A blocked temporary output path makes the terminal save fail while
16582        // preserving the already published reservation.
16583        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16584        let error = guard
16585            .finish(
16586                &mut runner.state,
16587                Err(
16588                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16589                        .into(),
16590                ),
16591            )
16592            .unwrap_err();
16593        assert_eq!(
16594            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16595            std::io::ErrorKind::PermissionDenied
16596        );
16597        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16598        let message = format!("{error:#}");
16599        assert!(message.contains("original refusal"));
16600        assert!(!message.contains("could not save terminal"));
16601        assert_eq!(
16602            error.chain().count(),
16603            2,
16604            "only startup attribution wraps the original error"
16605        );
16606        let saved = RunState::load(&runner.state.id).unwrap();
16607        assert_eq!(saved.status, RunStatus::Blocked);
16608        assert!(!saved.startup.unwrap().complete);
16609    }
16610
16611    #[test]
16612    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16613        ask_test_home();
16614        let mut runner = runner_at(RunStatus::Prep);
16615        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16616        let id = runner.state.id.clone();
16617        drop(guard);
16618        let saved = RunState::load(&id).unwrap();
16619        assert_eq!(saved.status, RunStatus::Blocked);
16620        assert!(!saved.startup.as_ref().unwrap().complete);
16621        assert!(
16622            saved
16623                .startup
16624                .unwrap()
16625                .failure
16626                .unwrap()
16627                .contains("interrupted")
16628        );
16629        assert!(Runner::resume(&id).is_err());
16630    }
16631
16632    #[test]
16633    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16634        ask_test_home();
16635        let mut runner = runner_at(RunStatus::Prep);
16636        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16637        let error = guard
16638            .finish(
16639                &mut runner.state,
16640                Err(
16641                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16642                        .into(),
16643                ),
16644            )
16645            .unwrap_err();
16646        assert_eq!(
16647            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16648            std::io::ErrorKind::PermissionDenied
16649        );
16650        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16651        let mut task = crate::queue::Task::new(
16652            "review".into(),
16653            "criteria".into(),
16654            runner.state.repo.clone(),
16655            crate::queue::Source::Human,
16656        );
16657        task.record_startup_failure(failure.clone());
16658        task.record_startup_failure(failure.clone());
16659        assert_eq!(task.startup_failures.len(), 1);
16660        assert!(task.runs.is_empty());
16661        assert_eq!(task.attempts, 0);
16662    }
16663
16664    #[cfg(unix)]
16665    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16666        ask_test_home();
16667        let tmp = tempfile::tempdir().unwrap();
16668        let repo = tmp.path().join("repo");
16669        std::fs::create_dir(&repo).unwrap();
16670        init_repo(&repo);
16671        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16672        let origin = tmp.path().join("origin.git");
16673        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16674            .await
16675            .unwrap();
16676        git::git(
16677            &repo,
16678            &["remote", "add", "origin", origin.to_str().unwrap()],
16679        )
16680        .await
16681        .unwrap();
16682        git::git(&repo, &["checkout", "-b", "fix/retained"])
16683            .await
16684            .unwrap();
16685        std::fs::write(repo.join("retained"), "retained change").unwrap();
16686        git::git(&repo, &["add", "."]).await.unwrap();
16687        git::git(&repo, &["commit", "-m", "retained implementation"])
16688            .await
16689            .unwrap();
16690        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16691        git::git(&repo, &["checkout", "main"]).await.unwrap();
16692        std::fs::write(repo.join("base-change"), "new base").unwrap();
16693        git::git(&repo, &["add", "."]).await.unwrap();
16694        git::git(&repo, &["commit", "-m", "base moved"])
16695            .await
16696            .unwrap();
16697        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16698        git::git(&repo, &["checkout", "fix/retained"])
16699            .await
16700            .unwrap();
16701        let mut runner = runner_at(RunStatus::Voting);
16702        runner.state.repo = repo.clone();
16703        runner.state.base_commit = base;
16704        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16705        runner.state.config.disk.min_free_bytes = 0;
16706        runner.state.config.graph.review_rounds = 1;
16707        runner.state.config.verify.e2e = vec!["true".into()];
16708        runner.state.config.verify.gate = vec!["true".into()];
16709        runner.state.candidates = vec![Candidate {
16710            index: 0,
16711            label: 'A',
16712            agent: "fixture".into(),
16713            branch: "fix/retained".into(),
16714            worktree: repo.clone(),
16715            summary: String::new(),
16716            stat: String::new(),
16717            files: 1,
16718            commits: 1,
16719            empty: false,
16720            failed: None,
16721            verified_noop: None,
16722            duration_ms: 0,
16723            folded: false,
16724        }];
16725        runner.state.tally = Some(Tally {
16726            first_choice: BTreeMap::new(),
16727            borda: BTreeMap::new(),
16728            winner: 'A',
16729            rankings: 0,
16730            unanimous_initial: true,
16731            deliberated: false,
16732            changed_votes: 0,
16733            unanimous_final: true,
16734            tie_break: None,
16735            judges: 0,
16736            present: 0,
16737            quorum: 0,
16738            met_quorum: true,
16739            uncontested: Some("review-only".into()),
16740        });
16741        let mut task = crate::queue::Task::new(
16742            "fixture".into(),
16743            "fixture".into(),
16744            repo,
16745            crate::queue::Source::Human,
16746        );
16747        task.link_run(&runner.state.id);
16748        crate::queue::Queue::open().put(&mut task).unwrap();
16749        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16750        runner.state.base_sync = Some(BaseSync {
16751            tip: String::new(),
16752            behind: 1,
16753            attempts: BASE_SYNC_ROUNDS,
16754            conflict: None,
16755            already_in: None,
16756        });
16757        runner.sync_to_base().await.unwrap();
16758        assert_eq!(runner.state.status, RunStatus::Blocked);
16759        (tmp, runner, head, task.id)
16760    }
16761
16762    #[cfg(unix)]
16763    #[tokio::test]
16764    async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
16765        for routed in [false, true] {
16766            let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
16767            let home = crate::run::home();
16768            let queue = crate::queue::Queue::open();
16769            let task = queue.get(&owner).unwrap();
16770            runner.state.task_instruction = Some(task.instruction.clone());
16771            runner.state.driver_exited = true;
16772            runner.state.save().unwrap();
16773            let questions = crate::ask::Questions::at(home.join("questions"));
16774            let mut q = crate::ask::Question::new(
16775                runner.state.id.clone(),
16776                "base-recovery".into(),
16777                "owner".into(),
16778                "Recover bounded base movement?".into(),
16779                String::new(),
16780                vec!["recover".into()],
16781            );
16782            q.actions.insert(
16783                "recover".into(),
16784                crate::ask::ChoiceAction::RecoverBase {
16785                    run: runner.state.id.clone(),
16786                    stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
16787                    episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
16788                },
16789            );
16790            q.routed_task = Some(owner.clone());
16791            q.answer(crate::ask::Answer::Choice("recover".into()))
16792                .unwrap();
16793            questions.put(&mut q).unwrap();
16794            crate::base_recovery_control::recover(
16795                &queue,
16796                &home,
16797                crate::base_recovery_control::Request {
16798                    run: runner.state.id.clone(),
16799                    head: head.clone(),
16800                    actor: "decisions".into(),
16801                    request: q.id.clone(),
16802                    decision: Some((owner.clone(), q.answered_at.unwrap())),
16803                    expected_stop: Some((
16804                        runner.state.base_sync_recovery.stop.clone().unwrap(),
16805                        runner.state.base_sync_recovery.episodes.last().unwrap().id,
16806                    )),
16807                },
16808            )
16809            .await
16810            .unwrap();
16811            if routed {
16812                questions
16813                    .update(&q.id, |q| {
16814                        q.routed_task = Some("another-owner".into());
16815                        Ok(())
16816                    })
16817                    .unwrap();
16818            } else {
16819                let mut task = queue.get(&owner).unwrap();
16820                task.hold_manual(Some("new owner decision".into()));
16821                // Keep the generation to exercise the separate owner-decision binding.
16822                task.status = crate::queue::TaskStatus::Queued;
16823                queue.put(&mut task).unwrap();
16824            }
16825            runner.state = RunState::load(&runner.state.id).unwrap();
16826            runner.sync_to_base().await.unwrap();
16827            let approval = runner
16828                .state
16829                .base_sync_recovery
16830                .authorizations
16831                .last()
16832                .unwrap();
16833            assert!(approval.consumed);
16834            assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
16835                "routing"
16836            } else {
16837                "owner decision"
16838            }));
16839            assert_eq!(
16840                runner.state.base_sync.as_ref().unwrap().attempts,
16841                BASE_SYNC_ROUNDS
16842            );
16843            assert_eq!(
16844                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16845                head
16846            );
16847        }
16848    }
16849
16850    #[cfg(unix)]
16851    #[tokio::test]
16852    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16853        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16854        if std::env::var_os(CHILD).is_none() {
16855            let home = tempfile::tempdir().unwrap();
16856            let output = std::process::Command::new(std::env::current_exe().unwrap())
16857                .args([
16858                    "--exact",
16859                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16860                    "--nocapture",
16861                ])
16862                .env(CHILD, "1")
16863                .env("MAGI_HOME", home.path())
16864                .env("MAGI_WORKER_HOME", home.path())
16865                .env("MAGI_WORKER_ID", "abcdef")
16866                .output()
16867                .unwrap();
16868            assert!(
16869                output.status.success()
16870                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16871                "{}\n{}",
16872                String::from_utf8_lossy(&output.stdout),
16873                String::from_utf8_lossy(&output.stderr)
16874            );
16875            return;
16876        }
16877        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16878        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16879        let home = crate::run::home();
16880        let queue = crate::queue::Queue::open();
16881        let mut owner = queue.get(&task).unwrap();
16882        owner.release_operator("test-release", None);
16883        owner.start(runner.state.id.clone());
16884        queue.put(&mut owner).unwrap();
16885        runner.state.driver_exited = true;
16886        runner.state.task_instruction = Some(owner.instruction.clone());
16887        runner.state.save().unwrap();
16888        let _claim = queue.claim(&task).unwrap();
16889        let reserve = |id: &str| {
16890            let directory = home.join("workers/jobs").join(id);
16891            std::fs::create_dir_all(&directory).unwrap();
16892            let job = crate::worker::Job {
16893                id: id.into(),
16894                sequence: 1,
16895                kind: "task".into(),
16896                key: format!("task:{task}"),
16897                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16898                generation: "fixture-generation".into(),
16899                version: None,
16900                status: crate::worker::Status::Running,
16901                result: None,
16902                pid: Some(std::process::id()),
16903                created_at: 0,
16904                finished_at: None,
16905            };
16906            std::fs::write(
16907                directory.join("job.json"),
16908                serde_json::to_vec(&job).unwrap(),
16909            )
16910            .unwrap();
16911            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16912        };
16913        let _own = reserve("abcdef");
16914        let other = reserve("fedcba");
16915        assert!(
16916            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16917                .await
16918                .is_err()
16919        );
16920        assert!(
16921            RunState::load(&runner.state.id)
16922                .unwrap()
16923                .base_sync_recovery
16924                .authorizations
16925                .is_empty()
16926        );
16927        drop(other);
16928        assert!(crate::daemon::is_working_on(
16929            &home,
16930            &runner.state.id,
16931            Timestamp::now()
16932        ));
16933        assert!(
16934            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16935        );
16936        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16937            .await
16938            .unwrap();
16939        assert!(
16940            RunState::load(&runner.state.id)
16941                .unwrap()
16942                .base_recovery_pending()
16943        );
16944    }
16945
16946    #[cfg(unix)]
16947    #[tokio::test]
16948    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16949        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16950        let home = crate::run::home();
16951        runner.state.driver_pid = Some(std::process::id());
16952        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16953        runner.state.driver_exited = false;
16954        runner.state.begin_pass();
16955        runner.state.save().unwrap();
16956        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16957        driver.record(&runner.state).unwrap();
16958        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16959        // Preserve the durable claim as if the process died before Drop saved.
16960        drop(driver);
16961        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16962        assert_eq!(
16963            runner.state.liveness_under(&home, false),
16964            crate::run::Liveness::Dead
16965        );
16966        let queue = crate::queue::Queue::open();
16967        let _claim = queue.claim(&task).unwrap();
16968        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16969            .await
16970            .unwrap();
16971        let disk = RunState::load(&runner.state.id).unwrap();
16972        assert!(!disk.driver_exited);
16973        assert!(disk.base_recovery_pending());
16974    }
16975
16976    #[cfg(unix)]
16977    #[tokio::test]
16978    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16979        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16980        if std::env::var_os(CHILD).is_none() {
16981            use std::os::unix::fs::PermissionsExt;
16982            let tmp = tempfile::tempdir().unwrap();
16983            let gh = tmp.path().join("gh");
16984            std::fs::write(&gh, r#"#!/bin/sh
16985case "$1 $2" in
16986  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16987  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16988  *) exit 2 ;;
16989esac
16990"#).unwrap();
16991            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16992            let mut paths = vec![tmp.path().to_path_buf()];
16993            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16994            let output = std::process::Command::new(std::env::current_exe().unwrap())
16995                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16996                .env(CHILD, "1")
16997                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16998                .env("PATH", std::env::join_paths(paths).unwrap())
16999                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
17000                .output().unwrap();
17001            assert!(
17002                output.status.success()
17003                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17004                "{}\n{}",
17005                String::from_utf8_lossy(&output.stdout),
17006                String::from_utf8_lossy(&output.stderr)
17007            );
17008            return;
17009        }
17010        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17011        let repo = runner.state.repo.clone();
17012        git::git(&repo, &["checkout", "main"]).await.unwrap();
17013        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
17014        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17015        git::git(&repo, &["checkout", "fix/retained"])
17016            .await
17017            .unwrap();
17018        runner.state.pr = Some(crate::run::PrRecord {
17019            url: "https://example.test/project/pull/1".into(),
17020            number: 1,
17021            state: "open".into(),
17022            checks: "unknown".into(),
17023            round: 0,
17024            rounds: 1,
17025            red_at_merge: Vec::new(),
17026        });
17027        runner.sync_to_base().await.unwrap();
17028        assert_eq!(runner.state.status, RunStatus::Blocked);
17029        assert_eq!(
17030            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17031            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17032        );
17033        assert!(runner.state.base_sync_closure_pending());
17034        assert!(crate::handover::can_resume(&runner.state));
17035        let queue = crate::queue::Queue::open();
17036        assert!(!crate::base_sync_recovery::held_task(
17037            &queue,
17038            &queue.get(&task).unwrap()
17039        ));
17040        assert!(runner.state.authorize_base_sync(&task).is_err());
17041        // Legacy closure stops have no typed stop but must retain the checkpoint.
17042        runner.state.base_sync_recovery.stop = None;
17043        runner.state.status = RunStatus::Voting;
17044        runner.state.gate_ran = true;
17045        runner.state.config.graph.review_rounds = 0;
17046        runner.state.save().unwrap();
17047        assert!(crate::handover::can_resume(&runner.state));
17048        runner.state = RunState::load(&runner.state.id).unwrap();
17049        runner.execute_graph().await.unwrap();
17050        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17051        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17052        assert_eq!(
17053            runner.state.base_sync.as_ref().unwrap().attempts,
17054            BASE_SYNC_ROUNDS
17055        );
17056        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17057        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17058        assert!(runner.state.reviews.is_empty());
17059        assert!(runner.state.gate.is_empty());
17060    }
17061
17062    #[cfg(unix)]
17063    #[tokio::test]
17064    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17065        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17066        runner.state.driver_exited = true;
17067        runner.state.save().unwrap();
17068        let home = crate::run::home();
17069        let queue = crate::queue::Queue::open();
17070        let _claim = queue.claim(&task).unwrap();
17071        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17072        assert!(
17073            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17074                .await
17075                .is_err()
17076        );
17077        drop(lease);
17078        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17079        assert!(
17080            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17081                .await
17082                .is_err()
17083        );
17084        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17085        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17086            .await
17087            .unwrap();
17088        let disk = RunState::load(&runner.state.id).unwrap();
17089        assert!(disk.base_recovery_pending());
17090        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17091        assert!(
17092            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17093                .await
17094                .is_err()
17095        );
17096        assert_eq!(
17097            RunState::load(&runner.state.id)
17098                .unwrap()
17099                .base_sync_recovery
17100                .authorizations
17101                .len(),
17102            1
17103        );
17104        let mut owner = queue.get(&task).unwrap();
17105        owner.runs.push("20260101-000000-ffff".into());
17106        queue.put(&mut owner).unwrap();
17107        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17108            .await
17109            .unwrap_err();
17110        assert!(error.to_string().contains("run task ownership changed"));
17111    }
17112
17113    #[cfg(unix)]
17114    #[tokio::test]
17115    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17116        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17117        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17118        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17119        legacy["status"] = serde_json::json!("voting");
17120        runner.state = serde_json::from_value(legacy).unwrap();
17121        runner.state.save().unwrap();
17122        for _ in 0..3 {
17123            runner.execute_graph().await.unwrap();
17124            assert_eq!(runner.state.status, RunStatus::Blocked);
17125            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17126            assert!(!crate::handover::can_resume(&runner.state));
17127            assert_eq!(
17128                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17129                head
17130            );
17131            assert!(runner.state.reviews.is_empty());
17132            assert!(!runner.state.gate_ran);
17133        }
17134        let notice = crate::notices::run_ended(&runner.state).unwrap();
17135        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17136        runner.state.begin_pass();
17137        runner.state.status = RunStatus::Voting;
17138        runner.deliberate().await.unwrap();
17139        runner.vote().await.unwrap();
17140        runner.finish_after_tally().await.unwrap();
17141        assert_eq!(runner.state.status, RunStatus::Blocked);
17142        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17143        assert_eq!(
17144            crate::notices::run_ended(&runner.state).unwrap().since,
17145            notice.since
17146        );
17147    }
17148
17149    #[cfg(unix)]
17150    #[tokio::test]
17151    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17152        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17153        for detail in [
17154            "fix/retained conflicts with origin/main and did not rebase: conflict",
17155            "rebased fix/retained locally but origin refused the push",
17156            "unknown failure",
17157        ] {
17158            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17159            runner.state.record_base_sync_stop(&head);
17160            assert!(runner.state.authorize_base_sync(&task).is_err());
17161            runner.state.status = RunStatus::Voting;
17162            runner.execute_graph().await.unwrap();
17163            assert_eq!(runner.state.status, RunStatus::Blocked);
17164            assert!(!crate::handover::can_resume(&runner.state));
17165            assert_eq!(
17166                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17167                head
17168            );
17169        }
17170    }
17171
17172    #[cfg(unix)]
17173    #[tokio::test]
17174    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17175        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17176        runner.state.authorize_base_sync(&task).unwrap();
17177        assert!(runner.state.consume_base_recovery(&head));
17178        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17179        runner.state.reserve_base_operation(&head, &tip);
17180        runner.state.save().unwrap();
17181        runner.state = RunState::load(&runner.state.id).unwrap();
17182        assert!(!runner.state.consume_base_recovery(&head));
17183        runner.sync_to_base().await.unwrap();
17184        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17185        assert_eq!(runner.state.base_episode_used(), 1);
17186        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17187        assert!(
17188            runner
17189                .state
17190                .base_sync_recovery
17191                .operations
17192                .last()
17193                .unwrap()
17194                .finished
17195        );
17196        assert!(!runner.state.base_sync_blocked());
17197        runner.sync_to_base().await.unwrap();
17198        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17199    }
17200
17201    #[cfg(unix)]
17202    #[tokio::test]
17203    async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17204        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17205        // Both historical diagnostic spellings retain an exhaustion identity.
17206        let original = runner
17207            .state
17208            .base_sync
17209            .as_ref()
17210            .unwrap()
17211            .conflict
17212            .clone()
17213            .unwrap();
17214        runner.state.base_sync.as_mut().unwrap().conflict = Some(
17215            original
17216                .replace("synchronization(s)", "rebase(s)")
17217                .replace("synchronizing", "rebasing"),
17218        );
17219        runner.state.base_sync_recovery = Default::default();
17220        runner.state.record_base_sync_stop(&head);
17221        assert_eq!(
17222            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17223            crate::base_sync_recovery::StopKind::Exhausted
17224        );
17225        runner.state.authorize_base_sync(&task).unwrap();
17226        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17227        let auth = legacy["base_sync_recovery"]["authorizations"][0]
17228            .as_object_mut()
17229            .unwrap();
17230        auth.remove("interval");
17231        auth.remove("scope");
17232        runner.state = serde_json::from_value(legacy).unwrap();
17233        assert!(runner.state.consume_base_recovery(&head));
17234        assert_eq!(runner.state.base_episode_used(), 3);
17235        runner.sync_to_base().await.unwrap();
17236        assert_eq!(runner.state.base_episode_used(), 4);
17237        runner.state.resource_block = Some(crate::run::ResourceBlock {
17238            node: "gate".into(),
17239            kind: "cache".into(),
17240            detail: "legacy wait".into(),
17241            at: Timestamp::now(),
17242            cause: crate::run::ResourceBlockCause::Unrecorded,
17243        });
17244        runner.state.save().unwrap();
17245        assert!(
17246            runner
17247                .state
17248                .base_sync_recovery
17249                .episodes
17250                .last()
17251                .unwrap()
17252                .boundary
17253                .is_none()
17254        );
17255        runner.state.resource_block = None;
17256        advance_base_fixture(&runner).await;
17257        runner.sync_to_base().await.unwrap();
17258        assert!(runner.state.base_sync_blocked());
17259        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17260    }
17261
17262    #[cfg(unix)]
17263    async fn advance_base_fixture(runner: &Runner) {
17264        let repo = &runner.state.repo;
17265        for args in [
17266            vec!["checkout", "main"],
17267            vec!["commit", "--allow-empty", "-m", "base advanced"],
17268            vec!["push", "origin", "main"],
17269            vec!["checkout", "fix/retained"],
17270        ] {
17271            git::git(repo, &args).await.unwrap();
17272        }
17273    }
17274
17275    #[cfg(unix)]
17276    fn fresh_base_interval_fixture(runner: &mut Runner) {
17277        runner.state.base_sync.as_mut().unwrap().attempts = 0;
17278        runner.state.base_sync.as_mut().unwrap().conflict = None;
17279        runner.state.base_sync_recovery = Default::default();
17280        runner.state.status = RunStatus::Voting;
17281    }
17282
17283    #[cfg(unix)]
17284    #[tokio::test]
17285    async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17286        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17287        fresh_base_interval_fixture(&mut runner);
17288        runner.state.begin_pass();
17289        runner.sync_to_base().await.unwrap();
17290        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17291        runner.state.resource_block = Some(crate::run::ResourceBlock {
17292            node: "gate".into(),
17293            kind: "build cache".into(),
17294            detail: "busy".into(),
17295            at: Timestamp::now(),
17296            cause: crate::run::ResourceBlockCause::Timeout {
17297                waited_secs: 1,
17298                limit_secs: 1,
17299            },
17300        });
17301        runner.state.save().unwrap();
17302        let queue = crate::queue::Queue::open();
17303        let mut owner = queue.get(&task).unwrap();
17304        runner.state.task_instruction = Some(owner.instruction.clone());
17305        owner.release_operator("fixture-owner", None);
17306        queue.put(&mut owner).unwrap();
17307        let approval = owner.resumption.clone().unwrap();
17308        runner.state.resource_block = None;
17309        runner.state.save().unwrap();
17310        runner.state = RunState::load(&runner.state.id).unwrap();
17311        advance_base_fixture(&runner).await;
17312        runner.sync_to_base().await.unwrap();
17313        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17314        assert_eq!(runner.state.base_episode_used(), 1);
17315        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17316        assert!(
17317            runner.state.base_sync_recovery.episodes[0]
17318                .boundary
17319                .as_ref()
17320                .unwrap()
17321                .consumed
17322        );
17323        assert!(
17324            !runner
17325                .state
17326                .renew_base_episode(&task, &approval, &head, "base")
17327        );
17328        for _ in 0..3 {
17329            owner.release_operator("fixture-owner", None);
17330            queue.put(&mut owner).unwrap();
17331            runner.state = RunState::load(&runner.state.id).unwrap();
17332            runner.state.begin_pass();
17333            runner.sync_to_base().await.unwrap();
17334            assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17335            assert_eq!(runner.state.base_episode_used(), 1);
17336        }
17337        runner.state.resource_block = Some(crate::run::ResourceBlock {
17338            node: "gate".into(),
17339            kind: "disk pressure".into(),
17340            detail: "free space required".into(),
17341            at: Timestamp::now(),
17342            cause: crate::run::ResourceBlockCause::Other,
17343        });
17344        runner.state.end_pass_parked(ParkCause::Interrupt {
17345            reason: "disk pressure".into(),
17346        });
17347        runner.state.save().unwrap();
17348        owner.release_operator("fixture-owner-after-disk", None);
17349        queue.put(&mut owner).unwrap();
17350        runner.state.resource_block = None;
17351        advance_base_fixture(&runner).await;
17352        runner.sync_to_base().await.unwrap();
17353        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17354        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17355        assert_eq!(runner.state.base_episode_used(), 1);
17356        assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17357    }
17358
17359    #[cfg(unix)]
17360    #[tokio::test]
17361    async fn base_recovery_continuous_advancement_stops_at_four() {
17362        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17363        fresh_base_interval_fixture(&mut runner);
17364        for spent in 1..=BASE_SYNC_ROUNDS {
17365            runner.sync_to_base().await.unwrap();
17366            assert!(!runner.state.base_sync_blocked());
17367            assert_eq!(runner.state.base_episode_used(), spent);
17368            advance_base_fixture(&runner).await;
17369        }
17370        runner.sync_to_base().await.unwrap();
17371        assert_eq!(
17372            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17373            crate::base_sync_recovery::StopKind::Exhausted
17374        );
17375        let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17376        assert!(stop.unverified_sync);
17377        let action = runner.state.base_sync_next_action();
17378        assert!(action.contains("never reached its gate"));
17379        assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17380        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17381        runner.state = RunState::load(&runner.state.id).unwrap();
17382        runner.sync_to_base().await.unwrap();
17383        assert_eq!(runner.state.base_episode_used(), 4);
17384        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17385    }
17386
17387    #[cfg(unix)]
17388    #[tokio::test]
17389    async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17390        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17391        runner.state.ensure_base_episode();
17392        // Verification progress alone is not evidence of a successful sync.
17393        runner.state.record_base_progress(&head);
17394        runner.state.base_sync_recovery.stop = None;
17395        runner.state.record_base_sync_stop(&head);
17396        assert!(
17397            !runner
17398                .state
17399                .base_sync_recovery
17400                .stop
17401                .as_ref()
17402                .unwrap()
17403                .unverified_sync
17404        );
17405
17406        runner.state.finish_base_operation(Some(&head));
17407        // A gate for another head cannot verify this synchronization.
17408        runner.state.gate_ran = true;
17409        runner.state.gate_head = Some("older-head".into());
17410        runner.state.base_sync_recovery.stop = None;
17411        runner.state.record_base_sync_stop(&head);
17412        runner.state.save().unwrap();
17413        runner.state = RunState::load(&runner.state.id).unwrap();
17414        assert!(
17415            runner
17416                .state
17417                .base_sync_recovery
17418                .stop
17419                .as_ref()
17420                .unwrap()
17421                .unverified_sync
17422        );
17423
17424        runner.state.gate_head = Some(head.clone());
17425        runner.state.base_sync_recovery.stop = None;
17426        runner.state.record_base_sync_stop(&head);
17427        assert!(
17428            !runner
17429                .state
17430                .base_sync_recovery
17431                .stop
17432                .as_ref()
17433                .unwrap()
17434                .unverified_sync
17435        );
17436        assert!(
17437            !runner
17438                .state
17439                .base_sync_next_action()
17440                .contains("never reached its gate")
17441        );
17442
17443        // Historical records cannot establish the missing synchronization evidence.
17444        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17445        legacy["base_sync_recovery"]["stop"]
17446            .as_object_mut()
17447            .unwrap()
17448            .remove("unverified_sync");
17449        legacy["base_sync_recovery"]["episodes"][0]
17450            .as_object_mut()
17451            .unwrap()
17452            .remove("last_sync_head");
17453        runner.state = serde_json::from_value(legacy).unwrap();
17454        assert!(
17455            !runner
17456                .state
17457                .base_sync_recovery
17458                .stop
17459                .as_ref()
17460                .unwrap()
17461                .unverified_sync
17462        );
17463        assert!(
17464            runner.state.base_sync_recovery.episodes[0]
17465                .last_sync_head
17466                .is_none()
17467        );
17468    }
17469
17470    #[cfg(unix)]
17471    #[tokio::test]
17472    async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17473        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17474        runner.state.authorize_base_sync(&task).unwrap();
17475        assert!(
17476            runner
17477                .state
17478                .base_sync_next_action()
17479                .contains("Approval recorded")
17480        );
17481        assert!(
17482            runner
17483                .state
17484                .base_sync_next_action()
17485                .contains("4/4 in episode 0, 4 lifetime")
17486        );
17487        assert!(runner.state.consume_base_recovery(&head));
17488        assert!(
17489            runner
17490                .state
17491                .base_sync_next_action()
17492                .contains("0/4 in episode 1, 4 lifetime")
17493        );
17494    }
17495
17496    #[cfg(unix)]
17497    #[tokio::test]
17498    async fn base_recovery_changed_head_invalidates_without_spending() {
17499        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17500        runner.state.authorize_base_sync(&task).unwrap();
17501        git::git(
17502            &runner.state.repo,
17503            &["commit", "--allow-empty", "-m", "external head change"],
17504        )
17505        .await
17506        .unwrap();
17507        runner.sync_to_base().await.unwrap();
17508        let a = runner
17509            .state
17510            .base_sync_recovery
17511            .authorizations
17512            .last()
17513            .unwrap();
17514        assert!(a.consumed);
17515        assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17516        runner.state = RunState::load(&runner.state.id).unwrap();
17517        assert!(
17518            runner
17519                .state
17520                .base_sync_next_action()
17521                .contains("Approval invalidated")
17522        );
17523        assert!(!runner.state.base_recovery_pending());
17524        assert!(runner.state.base_sync_blocked());
17525        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17526    }
17527
17528    #[cfg(unix)]
17529    #[tokio::test]
17530    async fn base_recovery_stale_fetched_base_invalidates_approval() {
17531        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17532        runner.state.driver_exited = true;
17533        runner.state.save().unwrap();
17534        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17535            .await
17536            .unwrap();
17537        runner.state = RunState::load(&runner.state.id).unwrap();
17538        advance_base_fixture(&runner).await;
17539        runner.sync_to_base().await.unwrap();
17540        let a = runner
17541            .state
17542            .base_sync_recovery
17543            .authorizations
17544            .last()
17545            .unwrap();
17546        assert!(
17547            a.consumed
17548                && a.invalidated
17549                    .as_deref()
17550                    .unwrap()
17551                    .contains("fetched base changed")
17552        );
17553        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17554        assert_eq!(
17555            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17556            head
17557        );
17558        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17559            .await
17560            .unwrap();
17561        runner.state = RunState::load(&runner.state.id).unwrap();
17562        runner.sync_to_base().await.unwrap();
17563        assert!(!runner.state.base_sync_blocked());
17564        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17565    }
17566
17567    #[cfg(unix)]
17568    #[tokio::test]
17569    async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17570        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17571        runner.state.authorize_base_sync(&task).unwrap();
17572        assert!(runner.state.consume_base_recovery(&head));
17573        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17574        runner.state.reserve_base_operation(&head, &tip);
17575        runner.state.save().unwrap();
17576        let scratch = runner.state.dir().join("base-sync");
17577        assert_eq!(
17578            crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17579                .await
17580                .unwrap(),
17581            crate::rebase::Rebased::Applied
17582        );
17583        runner.state = RunState::load(&runner.state.id).unwrap();
17584        runner.sync_to_base().await.unwrap();
17585        assert!(
17586            runner
17587                .state
17588                .base_sync_recovery
17589                .operations
17590                .last()
17591                .unwrap()
17592                .finished
17593        );
17594        assert_eq!(runner.state.base_episode_used(), 1);
17595        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17596        assert!(!runner.state.base_sync_blocked());
17597    }
17598
17599    #[cfg(unix)]
17600    #[tokio::test]
17601    async fn base_recovery_crash_preserves_untracked_files() {
17602        for ignored in [false, true] {
17603            let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17604            runner.state.authorize_base_sync(&task).unwrap();
17605            assert!(runner.state.consume_base_recovery(&head));
17606            let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17607            runner.state.reserve_base_operation(&head, &tip);
17608            runner.state.save().unwrap();
17609            let scratch = runner.state.dir().join("base-sync");
17610            assert_eq!(
17611                crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17612                    .await
17613                    .unwrap(),
17614                crate::rebase::Rebased::Applied
17615            );
17616            let repo = &runner.state.repo;
17617            let result = git::rev_parse(repo, "HEAD").await.unwrap();
17618            assert_ne!(result, head);
17619            if ignored {
17620                let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17621                    .await
17622                    .unwrap();
17623                std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17624            }
17625            std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17626            assert!(
17627                git::git_raw(repo, &["diff", "--quiet", &head])
17628                    .await
17629                    .unwrap()
17630                    .ok()
17631            );
17632            // Resume after the branch moved, before its checkout was reset.
17633            runner.state = RunState::load(&runner.state.id).unwrap();
17634            runner.sync_to_base().await.unwrap();
17635            assert!(runner.state.base_sync_blocked());
17636            assert!(!runner.state.base_sync_recovery.operations[0].finished);
17637            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17638            assert_eq!(
17639                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17640                result
17641            );
17642            assert_eq!(
17643                std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17644                "retained work\n"
17645            );
17646            assert!(
17647                git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17648                    .await
17649                    .unwrap()
17650                    .ok()
17651            );
17652        }
17653    }
17654
17655    #[cfg(unix)]
17656    #[tokio::test]
17657    async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17658        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17659        runner.state.authorize_base_sync(&task).unwrap();
17660        assert!(runner.state.consume_base_recovery(&head));
17661        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17662        runner.state.reserve_base_operation(&head, &tip);
17663        runner.state.save().unwrap();
17664        let scratch = runner.state.dir().join("base-sync");
17665        crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17666            .await
17667            .unwrap();
17668        git::sync_to_head(&runner.state.repo).await.unwrap();
17669        std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17670        git::git(&runner.state.repo, &["add", "foreign"])
17671            .await
17672            .unwrap();
17673        git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17674            .await
17675            .unwrap();
17676        let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17677        runner.state = RunState::load(&runner.state.id).unwrap();
17678        runner.sync_to_base().await.unwrap();
17679        assert!(runner.state.base_sync_blocked());
17680        assert!(
17681            !runner
17682                .state
17683                .base_sync_recovery
17684                .operations
17685                .last()
17686                .unwrap()
17687                .finished
17688        );
17689        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17690        assert_eq!(
17691            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17692            foreign_head
17693        );
17694    }
17695
17696    #[cfg(unix)]
17697    async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17698    {
17699        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17700        fresh_base_interval_fixture(&mut runner);
17701        let repo = &runner.state.repo;
17702        git::git(repo, &["checkout", "main"]).await.unwrap();
17703        std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17704        git::git(repo, &["add", "retained"]).await.unwrap();
17705        git::git(repo, &["commit", "-m", "upstream retained change"])
17706            .await
17707            .unwrap();
17708        git::git(repo, &["push", "origin", "main"]).await.unwrap();
17709        let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17710        git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17711        let scratch = runner.state.dir().join("base-sync");
17712        assert!(matches!(
17713            git::rebase_start(repo, &scratch, "fix/retained", &tip)
17714                .await
17715                .unwrap(),
17716            git::RebaseStart::Conflicted(_)
17717        ));
17718        runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17719            agent: "fixture".into(),
17720            paths: vec!["retained".into()],
17721            from: Some(head.clone()),
17722            stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17723            index_outside: None,
17724            finished: false,
17725            error: None,
17726            launcher: None,
17727            infra: None,
17728            outage: None,
17729        });
17730        (tmp, runner, head, tip, scratch)
17731    }
17732
17733    #[cfg(unix)]
17734    #[tokio::test]
17735    async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17736        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17737        // A resource stop from before reservations existed, with resolved
17738        // contents left for the launcher's ordinary staging continuation.
17739        std::fs::write(
17740            scratch.join("retained"),
17741            "upstream and retained resolution\n",
17742        )
17743        .unwrap();
17744        runner.state.resource_block = Some(crate::run::ResourceBlock {
17745            node: "rebase".into(),
17746            kind: "repository coordination".into(),
17747            detail: "busy".into(),
17748            at: Timestamp::now(),
17749            cause: crate::run::ResourceBlockCause::Other,
17750        });
17751        runner.state.save().unwrap();
17752        // Advance the remote without moving either checkout participating
17753        // in the standing rebase.
17754        let newer = git::git(
17755            &runner.state.repo,
17756            &[
17757                "commit-tree",
17758                "main^{tree}",
17759                "-p",
17760                "main",
17761                "-m",
17762                "base advanced during wait",
17763            ],
17764        )
17765        .await
17766        .unwrap();
17767        git::git(
17768            &runner.state.repo,
17769            &[
17770                "push",
17771                "origin",
17772                &format!("{}:refs/heads/main", newer.trim()),
17773            ],
17774        )
17775        .await
17776        .unwrap();
17777        runner.state = RunState::load(&runner.state.id).unwrap();
17778        runner.state.resource_block = None;
17779        runner.sync_to_base().await.unwrap();
17780        assert!(!runner.state.base_sync_blocked());
17781        let op = &runner.state.base_sync_recovery.operations[0];
17782        assert_eq!(op.head, head);
17783        assert_eq!(op.tip, tip);
17784        assert!(op.finished);
17785        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17786        assert_eq!(runner.state.rebase_fixes.len(), 1);
17787        let latest = git::rev_parse(&runner.state.repo, "origin/main")
17788            .await
17789            .unwrap();
17790        assert_ne!(latest, tip);
17791        runner.state = RunState::load(&runner.state.id).unwrap();
17792        runner.sync_to_base().await.unwrap();
17793        assert!(!runner.state.base_sync_blocked());
17794        assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17795        assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17796        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17797    }
17798
17799    #[cfg(unix)]
17800    #[tokio::test]
17801    async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17802        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17803        runner.state.reserve_base_operation(&head, &tip);
17804        runner.state.save().unwrap();
17805        std::fs::write(
17806            scratch.join("retained"),
17807            "upstream and retained resolution\n",
17808        )
17809        .unwrap();
17810        git::git(&scratch, &["add", "retained"]).await.unwrap();
17811        git::git(
17812            &scratch,
17813            &["-c", "core.editor=true", "rebase", "--continue"],
17814        )
17815        .await
17816        .unwrap();
17817        let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17818        assert!(
17819            !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17820                .await
17821                .unwrap()
17822                .is_empty()
17823        );
17824        assert_eq!(
17825            crate::rebase::finish(
17826                &mut runner.state,
17827                &scratch,
17828                "fix/retained",
17829                &head,
17830                &tip,
17831                &["retained".into()],
17832                ""
17833            )
17834            .await
17835            .unwrap(),
17836            crate::rebase::Rebased::Applied
17837        );
17838        assert!(!scratch.exists());
17839        // Simulate losing the caller before finish_base_operation/save.
17840        runner.state = RunState::load(&runner.state.id).unwrap();
17841        let op = &runner.state.base_sync_recovery.operations[0];
17842        assert!(!op.finished);
17843        assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17844        runner.sync_to_base().await.unwrap();
17845        assert!(!runner.state.base_sync_blocked());
17846        assert!(runner.state.base_sync_recovery.operations[0].finished);
17847        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17848        assert_eq!(runner.state.rebase_fixes.len(), 1);
17849        let winner = runner.state.winner().unwrap().clone();
17850        let wait = runner.repository_wait();
17851        assert!(
17852            verify_current_head(&mut runner.state, &winner, &wait)
17853                .await
17854                .unwrap()
17855        );
17856        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17857    }
17858
17859    #[cfg(unix)]
17860    #[tokio::test]
17861    async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17862        for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17863            let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17864            runner.state.reserve_base_operation(&head, &tip);
17865            runner.state.save().unwrap();
17866            std::fs::write(
17867                scratch.join("retained"),
17868                "upstream and retained resolution\n",
17869            )
17870            .unwrap();
17871            git::git(&scratch, &["add", "retained"]).await.unwrap();
17872            git::git(
17873                &scratch,
17874                &["-c", "core.editor=true", "rebase", "--continue"],
17875            )
17876            .await
17877            .unwrap();
17878            let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17879            assert!(!git::rebase_in_progress(&scratch).await);
17880            assert!(
17881                !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17882                    .await
17883                    .unwrap()
17884                    .is_empty()
17885            );
17886            match damage {
17887                Some("record") => {
17888                    let path =
17889                        git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17890                            .await
17891                            .unwrap();
17892                    std::fs::write(scratch.join(path), "").unwrap();
17893                }
17894                Some("foreign") => {
17895                    std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17896                    git::git(&scratch, &["add", "foreign"]).await.unwrap();
17897                    git::git(&scratch, &["commit", "-m", "foreign work"])
17898                        .await
17899                        .unwrap();
17900                }
17901                Some("dirty") => {
17902                    std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17903                }
17904                _ => {}
17905            }
17906            let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17907            // Crash immediately after Git continuation, before finish runs.
17908            runner.state = RunState::load(&runner.state.id).unwrap();
17909            assert!(
17910                runner.state.base_sync_recovery.operations[0]
17911                    .validated_head
17912                    .is_none()
17913            );
17914            runner.sync_to_base().await.unwrap();
17915            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17916            assert_eq!(runner.state.base_episode_used(), 1);
17917            assert_eq!(runner.state.rebase_fixes.len(), 1);
17918            assert_eq!(
17919                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17920                retained_head
17921            );
17922            if damage.is_some() {
17923                assert!(runner.state.base_sync_blocked());
17924                assert!(scratch.exists());
17925                assert!(!runner.state.base_sync_recovery.operations[0].finished);
17926                assert!(
17927                    runner.state.base_sync_recovery.operations[0]
17928                        .validated_head
17929                        .is_none()
17930                );
17931            } else {
17932                assert!(!runner.state.base_sync_blocked());
17933                assert!(!scratch.exists());
17934                assert!(runner.state.base_sync_recovery.operations[0].finished);
17935                assert_eq!(
17936                    runner.state.base_sync_recovery.operations[0]
17937                        .validated_head
17938                        .as_deref(),
17939                    Some(result.as_str())
17940                );
17941                let winner = runner.state.winner().unwrap().clone();
17942                let wait = runner.repository_wait();
17943                assert!(
17944                    verify_current_head(&mut runner.state, &winner, &wait)
17945                        .await
17946                        .unwrap()
17947                );
17948                assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17949                runner.state = RunState::load(&runner.state.id).unwrap();
17950                runner.sync_to_base().await.unwrap();
17951                assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17952            }
17953        }
17954    }
17955
17956    #[cfg(unix)]
17957    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17958        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17959        // This is an ordinary gate stop, not an authorized base-budget recovery.
17960        runner.state.prep_complete = Some(true);
17961        runner.state.base_sync = None;
17962        runner.state.base_sync_recovery = Default::default();
17963        runner.state.config.graph.gate_fix_rounds = 1;
17964        let mut review = review_round(true, 0, 1, 1, false, true);
17965        review.head = head.clone();
17966        review.verified_head = Some(head.clone());
17967        runner.state.reviews = vec![review];
17968        runner.state.gate = vec![CommandOutcome {
17969            command: "old gate".into(),
17970            code: Some(1),
17971            output_tail: "old failure".into(),
17972            duration_ms: 0,
17973            resource_blocked: false,
17974        }];
17975        runner.state.gate_head = Some(head.clone());
17976        runner.state.gate_ran = true;
17977        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17978            agent: "fixture".into(),
17979            failed: runner.state.gate.clone(),
17980            notes: "spent".into(),
17981            committed: false,
17982            error: None,
17983        }];
17984        runner.state.land_approval = Some(crate::run::LandApproval {
17985            question: "old approval".into(),
17986            head: head.clone(),
17987        });
17988        runner.state.status = RunStatus::Blocked;
17989        (tmp, runner, head)
17990    }
17991
17992    #[cfg(unix)]
17993    #[tokio::test]
17994    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17995        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17996            let (tmp, mut runner, old) = resumed_gate_fixture().await;
17997            let phases = tmp.path().join("phases");
17998            runner.state.config.verify.e2e = vec![format!(
17999                "echo e2e >> '{}'; {}",
18000                phases.display(),
18001                if e2e_ok {
18002                    "true"
18003                } else {
18004                    "echo new-e2e-failure; false"
18005                }
18006            )];
18007            runner.state.config.verify.gate = vec![format!(
18008                "echo gate >> '{}'; {}",
18009                phases.display(),
18010                if gate_ok {
18011                    "true"
18012                } else {
18013                    "echo new-gate-failure; false"
18014                }
18015            )];
18016            runner.state.save().unwrap();
18017            runner.state = RunState::load(&runner.state.id).unwrap();
18018            runner.sync_to_base().await.unwrap();
18019            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18020            assert_ne!(head, old);
18021            // Reentry after Git updated but before verification was invalidated.
18022            runner.state = RunState::load(&runner.state.id).unwrap();
18023            runner.execute_graph().await.unwrap();
18024            assert_eq!(runner.state.gate_fixes.len(), 1);
18025            assert_eq!(runner.state.reviews.len(), 1);
18026            assert_eq!(runner.state.reviews[0].head, old);
18027            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18028            assert_eq!(runner.state.verification_history.len(), 1);
18029            let history = &runner.state.verification_history[0];
18030            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18031            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18032            assert!(history.archived_at.is_some());
18033            assert_eq!(history.fixes_spent, Some(1));
18034            assert_eq!(history.gate[0].output_tail, "old failure");
18035            let archive_events: Vec<_> = runner
18036                .state
18037                .events
18038                .iter()
18039                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18040                .collect();
18041            assert_eq!(archive_events.len(), 1);
18042            assert!(archive_events[0].message.contains(&short(&old)));
18043            assert!(archive_events[0].message.contains(&short(&head)));
18044            assert!(
18045                archive_events[0]
18046                    .message
18047                    .contains("1 gate-fix round(s) already spent stay spent")
18048            );
18049            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18050            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18051            if e2e_ok {
18052                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18053                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18054            } else {
18055                assert!(!runner.state.gate_ran);
18056            }
18057            assert_eq!(
18058                runner.state.status,
18059                if e2e_ok && gate_ok {
18060                    RunStatus::Ready
18061                } else {
18062                    RunStatus::Blocked
18063                }
18064            );
18065            if !e2e_ok || !gate_ok {
18066                let cause = runner.state.verification_blocking_cause().unwrap();
18067                assert!(cause.contains(&head));
18068                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18069                // A later successful progress event must not become the cause.
18070                runner.state.event("land", "rebased successfully");
18071                let notice = crate::notices::run_ended(&runner.state).unwrap();
18072                assert!(!format!("{notice:?}").contains("rebased successfully"));
18073                assert!(runner.state.merge.is_none());
18074            }
18075            let before = std::fs::read_to_string(&phases).unwrap();
18076            runner.state.save().unwrap();
18077            runner.state = RunState::load(&runner.state.id).unwrap();
18078            runner.review_loop().await.unwrap();
18079            runner.gate().await.unwrap();
18080            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18081            assert_eq!(runner.state.verification_history.len(), 1);
18082            let history = &runner.state.verification_history[0];
18083            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18084            assert!(history.archived_at.is_some());
18085            assert_eq!(history.fixes_spent, Some(1));
18086        }
18087    }
18088
18089    #[cfg(unix)]
18090    #[tokio::test]
18091    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18092        let (tmp, mut runner, head) = resumed_gate_fixture().await;
18093        let marker = tmp.path().join("gate-ran");
18094        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18095        runner.review_loop().await.unwrap();
18096        runner.gate().await.unwrap();
18097        assert_eq!(runner.state.status, RunStatus::Blocked);
18098        assert!(!marker.exists());
18099        assert!(runner.state.verification_history.is_empty());
18100        runner.state.gate_head = None;
18101        runner.gate().await.unwrap();
18102        assert!(marker.exists());
18103        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18104        assert_eq!(runner.state.verification_history.len(), 1);
18105        assert_eq!(runner.state.gate_fixes.len(), 1);
18106    }
18107
18108    #[cfg(unix)]
18109    #[tokio::test]
18110    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18111        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18112        runner.sync_to_base().await.unwrap();
18113        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18114        runner.state.reconcile_verification_head(&head);
18115        // Both crash-before-command and resource wait are unresolved evidence.
18116        for outcomes in [
18117            Vec::new(),
18118            vec![CommandOutcome {
18119                command: "true".into(),
18120                code: None,
18121                output_tail: "cache unavailable".into(),
18122                duration_ms: 0,
18123                resource_blocked: true,
18124            }],
18125        ] {
18126            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18127                head: head.clone(),
18128                passed: false,
18129                outcomes,
18130            });
18131            runner.state.gate_ran = false;
18132            runner.state.gate.clear();
18133            runner.state.save().unwrap();
18134            runner.state = RunState::load(&runner.state.id).unwrap();
18135            runner.merge().await.unwrap();
18136            assert!(runner.state.merge.is_none());
18137            runner.review_loop().await.unwrap();
18138            runner.gate().await.unwrap();
18139            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18140            assert!(runner.state.gate_allows_progress());
18141            assert_eq!(runner.state.gate_fixes.len(), 1);
18142            assert!(runner.state.merge.is_none());
18143        }
18144        // Moving the winner after a successful gate cannot publish an unchecked SHA.
18145        git::git(
18146            &runner.state.repo,
18147            &["commit", "--allow-empty", "-m", "external update"],
18148        )
18149        .await
18150        .unwrap();
18151        runner.merge().await.unwrap();
18152        assert!(runner.state.merge.is_none());
18153        assert_eq!(runner.state.status, RunStatus::Blocked);
18154    }
18155
18156    #[cfg(unix)]
18157    #[tokio::test]
18158    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18159        for waiting_on_gate in [false, true] {
18160            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18161            runner.sync_to_base().await.unwrap();
18162            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18163            runner.state.reconcile_verification_head(&head);
18164            if waiting_on_gate {
18165                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18166                    head: head.clone(),
18167                    passed: true,
18168                    outcomes: Vec::new(),
18169                });
18170            }
18171            let cache = tmp.path().join("cache");
18172            runner.state.config.verify.e2e =
18173                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18174            runner.state.config.verify.gate = vec!["true".into()];
18175            runner.state.config.graph.timeout_resource_wait = 1;
18176            let owner =
18177                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18178            let guard =
18179                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18180                    crate::cache::AcquireOutcome::Acquired(g) => g,
18181                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18182                };
18183            runner.review_loop().await.unwrap();
18184            runner.gate().await.unwrap();
18185            assert!(!runner.state.gate_ran);
18186            assert_eq!(runner.state.gate_fixes.len(), 1);
18187            assert!(runner.state.resource_block.is_some());
18188            runner.merge().await.unwrap();
18189            assert!(runner.state.merge.is_none());
18190            drop(guard);
18191            runner.state = RunState::load(&runner.state.id).unwrap();
18192            // The driver clears the previous pass's resource stop on resume.
18193            runner.state.resource_block = None;
18194            runner.review_loop().await.unwrap();
18195            runner.gate().await.unwrap();
18196            assert!(runner.state.gate_allows_progress());
18197            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18198            assert_eq!(runner.state.gate_fixes.len(), 1);
18199        }
18200    }
18201
18202    #[cfg(unix)]
18203    #[tokio::test]
18204    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18205        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18206        runner.sync_to_base().await.unwrap();
18207        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18208        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18209        // Verification passes before pre_gate and fails on the mechanically changed head.
18210        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18211        runner.review_loop().await.unwrap();
18212        runner.gate().await.unwrap();
18213        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18214        assert_ne!(before, after);
18215        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18216        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18217        assert!(!runner.state.gate_allows_progress());
18218        runner.merge().await.unwrap();
18219        assert!(runner.state.merge.is_none());
18220        assert!(
18221            runner
18222                .state
18223                .verification_history
18224                .iter()
18225                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18226        );
18227
18228        // An incomplete review is still an independent hold on a changed head.
18229        runner.state.reviews[0].clean = false;
18230        runner.state.reviews[0].answered = 0;
18231        runner.state.config.verify.e2e = vec!["true".into()];
18232        runner.review_loop().await.unwrap();
18233        runner.gate().await.unwrap();
18234        runner.merge().await.unwrap();
18235        assert_eq!(runner.state.status, RunStatus::Blocked);
18236        assert!(runner.state.merge.is_none());
18237        assert!(
18238            runner
18239                .state
18240                .verification_blocking_cause()
18241                .unwrap()
18242                .contains("review blocked")
18243        );
18244    }
18245
18246    #[cfg(unix)]
18247    #[tokio::test]
18248    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18249        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18250        runner.state.gate_head = None;
18251        runner.state.config.verify.gate =
18252            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18253        runner.review_loop().await.unwrap();
18254        runner.gate().await.unwrap();
18255        assert_ne!(
18256            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18257            old
18258        );
18259        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18260        assert!(!runner.state.gate_allows_progress());
18261        assert_eq!(runner.state.status, RunStatus::Blocked);
18262        runner.merge().await.unwrap();
18263        assert!(runner.state.merge.is_none());
18264    }
18265
18266    #[cfg(unix)]
18267    #[tokio::test]
18268    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18269        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18270        let mut old_review = review_round(true, 0, 1, 1, false, true);
18271        old_review.head = head.clone();
18272        old_review.verified_head = Some(head.clone());
18273        runner.state.reviews = vec![old_review];
18274        runner.state.gate = vec![CommandOutcome {
18275            command: "old gate".into(),
18276            code: None,
18277            output_tail: "timed out after 2400s".into(),
18278            duration_ms: 2_400_000,
18279            resource_blocked: false,
18280        }];
18281        runner.state.gate_ran = true;
18282        runner.state.gate_head = Some(head.clone());
18283        runner.state.land_approval = Some(crate::run::LandApproval {
18284            question: "old-head-approval".into(),
18285            head: head.clone(),
18286        });
18287        runner.state.authorize_base_sync(&task).unwrap();
18288        runner.sync_to_base().await.unwrap();
18289        assert!(!runner.state.base_sync_blocked());
18290        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18291        assert_ne!(head, fresh);
18292        assert_eq!(runner.state.reviews.len(), 1);
18293        assert!(!runner.state.gate_ran);
18294        let saved = &runner.state.base_sync_recovery.authorizations[0];
18295        assert_eq!(saved.reviews[0].head, head);
18296        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18297        assert!(saved.gate_ran);
18298        assert!(!saved.gate[0].ok());
18299        runner.review_loop().await.unwrap();
18300        runner.gate().await.unwrap();
18301        assert_eq!(runner.state.reviews[0].head, head);
18302        assert_eq!(
18303            runner.state.reviews[0].verified_head.as_deref(),
18304            Some(head.as_str())
18305        );
18306        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18307        assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18308        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18309        assert!(runner.state.gate_ran);
18310        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18311        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18312        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18313        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18314        assert_eq!(
18315            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18316            "retained change"
18317        );
18318    }
18319
18320    #[cfg(unix)]
18321    #[tokio::test]
18322    async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18323        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18324        let queue = crate::queue::Queue::open();
18325        let mut owner = queue.get(&task).unwrap();
18326        owner.release_operator("test-resume", None);
18327        queue.put(&mut owner).unwrap();
18328        let generation = owner.resumption.as_ref().unwrap().generation;
18329        runner.state.task_instruction = Some(owner.instruction.clone());
18330        runner.state.driver_exited = true;
18331        runner.state.save().unwrap();
18332        let home = crate::run::home();
18333        let _claim = queue.claim(&task).unwrap();
18334        owner.resumption.as_mut().unwrap().at =
18335            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18336        queue.put(&mut owner).unwrap();
18337        assert!(
18338            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18339                .await
18340                .is_err(),
18341            "an approval from before the stop grants nothing"
18342        );
18343        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18344        queue.put(&mut owner).unwrap();
18345        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18346            .await
18347            .unwrap();
18348        assert!(
18349            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18350                .await
18351                .is_err()
18352        );
18353        runner.state = RunState::load(&runner.state.id).unwrap();
18354        assert_eq!(
18355            runner.state.base_sync_recovery.authorizations[0].generation,
18356            Some(generation)
18357        );
18358        runner.sync_to_base().await.unwrap();
18359        let repo = runner.state.repo.clone();
18360        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18361        git::git(&repo, &["checkout", "main"]).await.unwrap();
18362        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18363        git::git(&repo, &["add", "."]).await.unwrap();
18364        git::git(&repo, &["commit", "-m", "move base again"])
18365            .await
18366            .unwrap();
18367        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18368        git::git(&repo, &["checkout", "fix/retained"])
18369            .await
18370            .unwrap();
18371        runner.sync_to_base().await.unwrap();
18372        assert!(!runner.state.base_sync_blocked());
18373        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18374        assert_eq!(runner.state.base_episode_used(), 2);
18375        let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18376        assert_ne!(fresh, head);
18377        assert!(!runner.state.base_recovery_pending());
18378        // Even replaying the generation with a later timestamp grants nothing.
18379        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18380        queue.put(&mut owner).unwrap();
18381        assert!(
18382            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18383                .await
18384                .is_err()
18385        );
18386        runner.state = RunState::load(&runner.state.id).unwrap();
18387        runner.sync_to_base().await.unwrap();
18388        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18389        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18390        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18391    }
18392
18393    #[cfg(unix)]
18394    #[tokio::test]
18395    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18396        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18397        runner.state.base_sync = None;
18398        runner.state.status = RunStatus::Blocked;
18399        let mut second = runner.state.candidates[0].clone();
18400        second.label = 'B';
18401        runner.state.candidates.push(second);
18402        assert!(runner.state.votes.is_empty());
18403        assert!(runner.state.deliberation.is_empty());
18404        runner.deliberate().await.unwrap();
18405        runner.vote().await.unwrap();
18406        assert_eq!(runner.state.status, RunStatus::Blocked);
18407        assert!(runner.state.votes.is_empty());
18408        assert!(runner.state.deliberation.is_empty());
18409    }
18410
18411    /// A minimal, git-free `Runner` at a given status — `settle_questions`
18412    /// reads nothing else off it.
18413    fn runner_at(status: RunStatus) -> Runner {
18414        let mut state = RunState::new(
18415            PathBuf::from("/nonexistent/repo"),
18416            "main".to_owned(),
18417            "deadbeef".to_owned(),
18418            "task".to_owned(),
18419            Config::default(),
18420        );
18421        // These fixtures test execution flow, independent of host disk capacity.
18422        state.config.disk.min_free_bytes = 0;
18423        // These fixtures share one home and save concurrently. Short clock
18424        // IDs can collide, making an ownership test read another test's run.
18425        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18426        state.id = format!(
18427            "20000101-000000-{:04x}",
18428            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18429        );
18430        state.status = status;
18431        Runner {
18432            state,
18433            roles: ResolvedRoles {
18434                implementers: Vec::new(),
18435                judges: Vec::new(),
18436                reviewers: Vec::new(),
18437                fixer: None,
18438                conductor: conductor(),
18439                implementer_roster: Vec::new(),
18440                judge_roster: Vec::new(),
18441                reviewer_roster: Vec::new(),
18442            },
18443            sem: Arc::new(Semaphore::new(1)),
18444            pause: Pause::new(),
18445            interrupt: Pause::new(),
18446        }
18447    }
18448
18449    #[tokio::test]
18450    async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18451        crate::run::pin_test_home();
18452        let temp = tempfile::tempdir().unwrap();
18453        let root = temp.path().canonicalize().unwrap();
18454        let repo = root.join("repo");
18455        std::fs::create_dir(&repo).unwrap();
18456        init_repo(&repo);
18457        git::git(&repo, &["config", "core.autocrlf", "false"])
18458            .await
18459            .unwrap();
18460        let mut runner = runner_at(RunStatus::Blocked);
18461        runner.state.repo = repo.clone();
18462        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18463        runner.state.config.agents = vec![conductor()];
18464        runner.state.config.graph.selected_implementer = Some("conductor".into());
18465        runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18466        let (shell, command, setup_output) = if cfg!(windows) {
18467            (
18468                vec!["cmd".into(), "/C".into()],
18469                "echo ready>seat-ready",
18470                "ready\r\n",
18471            )
18472        } else {
18473            (
18474                vec!["sh".into(), "-c".into()],
18475                "printf ready > seat-ready",
18476                "ready",
18477            )
18478        };
18479        runner.state.config.verify.shell = Some(shell);
18480        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18481            run: Some(command.into()),
18482            ..Default::default()
18483        }];
18484        runner.state.save().unwrap();
18485        let path = runner.state.worktree_root().join("judge-1");
18486        git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18487            .await
18488            .unwrap();
18489        std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18490        git::git(&path, &["add", "."]).await.unwrap();
18491        let tree = git::git(&path, &["write-tree"]).await.unwrap();
18492        let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18493            .await
18494            .unwrap();
18495        git::git(&path, &["checkout", "--detach", &head])
18496            .await
18497            .unwrap();
18498        let evidence = crate::fold_evidence::snapshot(
18499            &runner.state,
18500            &crate::run::home().canonicalize().unwrap(),
18501            &head,
18502            64 * 1024 * 1024,
18503        )
18504        .await
18505        .unwrap();
18506        let mut contents = BTreeMap::new();
18507        contents.insert(
18508            repo.join("logs/HEAD"),
18509            format!("{head} {head} fixture\n").into_bytes(),
18510        );
18511        let record = serde_json::json!({
18512            "version":1, "run":runner.state.id, "source":path, "head":head,
18513            "evidence":evidence, "admin":{}, "admin_contents":contents,
18514            "history":{head.clone():evidence}, "phase":"evicted"
18515        });
18516        std::fs::write(
18517            runner.state.dir().join("artifacts/eviction-judge-1.json"),
18518            serde_json::to_vec(&record).unwrap(),
18519        )
18520        .unwrap();
18521        assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18522        git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18523            .await
18524            .unwrap();
18525        git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18526        assert!(
18527            !git::git_raw(&repo, &["cat-file", "-e", &head])
18528                .await
18529                .unwrap()
18530                .ok()
18531        );
18532        let mut resumed = Runner::resume(&runner.state.id).unwrap();
18533        let error = resumed.execute().await.unwrap_err();
18534        assert!(
18535            error
18536                .to_string()
18537                .contains("selected implementation is held"),
18538            "{error:#}"
18539        );
18540        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18541        assert_eq!(
18542            std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18543            setup_output
18544        );
18545        assert_eq!(
18546            std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18547            "unrelated seat"
18548        );
18549        let error = resumed.execute().await.unwrap_err();
18550        assert!(
18551            error
18552                .to_string()
18553                .contains("selected implementation is held"),
18554            "{error:#}"
18555        );
18556        std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18557        git::git(&path, &["add", "."]).await.unwrap();
18558        let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18559        let next = git::git(
18560            &path,
18561            &[
18562                "commit-tree",
18563                &next_tree,
18564                "-p",
18565                &head,
18566                "-m",
18567                "next review checkpoint",
18568            ],
18569        )
18570        .await
18571        .unwrap();
18572        git::git(&path, &["checkout", "--detach", &next])
18573            .await
18574            .unwrap();
18575        resumed.state.reviews.push(
18576            serde_json::from_value(serde_json::json!({
18577                "round":1, "head":next, "reviews":[]
18578            }))
18579            .unwrap(),
18580        );
18581        resumed.state.save().unwrap();
18582        let error = resumed.execute().await.unwrap_err();
18583        assert!(
18584            error
18585                .to_string()
18586                .contains("selected implementation is held"),
18587            "{error:#}"
18588        );
18589        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18590    }
18591
18592    #[tokio::test]
18593    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18594        crate::run::pin_test_home();
18595        let mut runner = runner_at(RunStatus::Gating);
18596        runner.state.save().unwrap();
18597        let id = runner.state.id.clone();
18598        let passes = runner.state.passes.len();
18599        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18600        let mut claiming = Box::pin(runner.claim_driver());
18601        std::future::poll_fn(|cx| {
18602            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18603            std::task::Poll::Ready(())
18604        })
18605        .await;
18606        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18607        drop(owner);
18608        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18609            .await
18610            .unwrap()
18611            .unwrap();
18612        assert_eq!(execution.runner.state.passes.len(), passes + 1);
18613        assert!(execution.runner.state.resource_block.is_none());
18614        assert_eq!(
18615            execution.runner.state.liveness(false),
18616            crate::run::Liveness::Live
18617        );
18618        drop(execution);
18619        assert!(RunState::load(&id).unwrap().driver_exited);
18620    }
18621
18622    #[tokio::test]
18623    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18624        crate::run::pin_test_home();
18625        let mut runner = runner_at(RunStatus::Gating);
18626        runner.state.save().unwrap();
18627        // run.json can commit, but the subsequent lease metadata write fails.
18628        // Cleanup must run after releasing the short ownership lock.
18629        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18630        assert!(runner.claim_driver().await.is_err());
18631        assert!(runner.state.driver_exited);
18632        assert_eq!(runner.state.driver_pid, None);
18633        let disk = RunState::load(&runner.state.id).unwrap();
18634        assert!(disk.driver_exited);
18635        assert_eq!(disk.driver_pid, None);
18636        assert_eq!(disk.status, RunStatus::Gating);
18637    }
18638
18639    #[tokio::test]
18640    async fn an_early_graph_error_releases_its_driver() {
18641        crate::run::pin_test_home();
18642        let mut runner = runner_at(RunStatus::Blocked);
18643        runner.state.config.graph.selected_implementer = Some("selected".into());
18644        runner.state.save().unwrap();
18645        assert!(runner.execute().await.is_err());
18646        let disk = RunState::load(&runner.state.id).unwrap();
18647        assert!(disk.driver_exited);
18648        assert_eq!(disk.driver_pid, None);
18649        assert_eq!(disk.status, RunStatus::Blocked);
18650    }
18651
18652    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18653        crate::run::MergeRepair {
18654            question: "repair-question".into(),
18655            head: "old-head".into(),
18656            pr: "https://github.com/o/r/pull/42".into(),
18657            reviews: vec![review_round(false, 1, 1, 1, false, true)],
18658            gate: vec![outcome(Some(0), false)],
18659            gate_head: Some("old-head".into()),
18660            gate_fixes: Vec::new(),
18661            pre_gate: Vec::new(),
18662            phase,
18663            failure: None,
18664        }
18665    }
18666
18667    #[tokio::test]
18668    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18669        crate::run::pin_test_home();
18670        let mut runner = runner_at(RunStatus::Landing);
18671        runner.state.merge_repairs.push(merge_repair_request(
18672            crate::run::MergeRepairPhase::Requested,
18673        ));
18674        runner.state.save().unwrap();
18675        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18676        assert!(
18677            runner
18678                .execute()
18679                .await
18680                .unwrap_err()
18681                .to_string()
18682                .contains("already running")
18683        );
18684        assert_eq!(
18685            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18686            crate::run::MergeRepairPhase::Requested
18687        );
18688        drop(claim);
18689        let mut disk = runner.state.clone();
18690        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18691        disk.save().unwrap();
18692        assert!(
18693            runner
18694                .execute()
18695                .await
18696                .unwrap_err()
18697                .to_string()
18698                .contains("decision changed")
18699        );
18700        assert_eq!(
18701            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18702            crate::run::MergeRepairPhase::Complete
18703        );
18704    }
18705
18706    #[test]
18707    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18708        crate::run::pin_test_home();
18709        let mut runner = runner_at(RunStatus::Landing);
18710        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18711        runner.state.reviews = request.reviews.clone();
18712        runner.state.gate = request.gate.clone();
18713        runner.state.gate_ran = true;
18714        runner.state.land_approval = Some(crate::run::LandApproval {
18715            question: "old".into(),
18716            head: "old-head".into(),
18717        });
18718        runner.state.merge_repairs.push(request);
18719        runner.begin_merge_repair_review(0).unwrap();
18720        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18721        assert!(!runner.state.gate_status().ok());
18722        assert!(runner.state.land_approval.is_none());
18723        let disk = RunState::load(&runner.state.id).unwrap();
18724        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18725        assert!(disk.merge_repairs[0].gate[0].ok());
18726        assert_eq!(
18727            disk.merge_repairs[0].phase,
18728            crate::run::MergeRepairPhase::Reviewing
18729        );
18730        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18731        assert!(disk.reviews.is_empty());
18732    }
18733
18734    #[tokio::test]
18735    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18736        crate::run::pin_test_home();
18737        for phase in [
18738            crate::run::MergeRepairPhase::Fixing,
18739            crate::run::MergeRepairPhase::Requested,
18740        ] {
18741            let mut runner = runner_at(RunStatus::Landing);
18742            runner.state.config.graph.land_rounds = 0;
18743            runner.state.merge_repairs.push(merge_repair_request(phase));
18744            runner.state.save().unwrap();
18745            assert!(!runner.repair_merge_review().await.unwrap());
18746            assert_eq!(runner.state.status, RunStatus::Blocked);
18747            assert_eq!(
18748                runner.state.merge_repairs[0].phase,
18749                crate::run::MergeRepairPhase::Failed
18750            );
18751            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18752            assert!(
18753                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18754                    "interrupted"
18755                } else {
18756                    "budget exhausted"
18757                })
18758            );
18759            assert!(runner.state.merge.is_none());
18760        }
18761    }
18762
18763    /// `park_here` folding in the reason `Pause::park_because` recorded -
18764    /// this is what lets an operator reading a run's events tell an
18765    /// interrupt-driven park from an ordinary shutdown park.
18766    #[test]
18767    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18768        crate::run::pin_test_home();
18769        let mut runner = runner_at(RunStatus::Implementing);
18770        let interrupt = Pause::new();
18771        runner.watch_interrupt(interrupt.clone());
18772
18773        interrupt.park_because("task a1b2 asked to run first");
18774
18775        assert!(runner.park_here().expect("park_here"));
18776        assert!(runner.state.parked);
18777        let last = runner.state.events.last().expect("a park event");
18778        assert_eq!(last.node, "park");
18779        assert!(
18780            last.message.contains("task a1b2 asked to run first"),
18781            "expected the interrupt reason in {:?}",
18782            last.message
18783        );
18784    }
18785
18786    /// The cause is stored on the pass when it parks, and the first cause a
18787    /// shared pause was given is the one that stays.
18788    #[test]
18789    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18790        crate::run::pin_test_home();
18791        let mut runner = runner_at(RunStatus::Implementing);
18792        runner.state.begin_pass();
18793        let shutdown = Pause::new();
18794        runner.on_pause(shutdown.clone());
18795        let upgrade = ParkCause::Upgrade {
18796            from: "0.121.0".to_owned(),
18797            to: Some("0.121.1".to_owned()),
18798        };
18799        shutdown.park_for(upgrade.clone());
18800        shutdown.park_for(ParkCause::Operator);
18801
18802        assert!(runner.park_here().expect("park_here"));
18803        assert_eq!(
18804            runner.state.passes[0].end,
18805            Some(crate::run::PassEnd::Parked { cause: upgrade })
18806        );
18807        let last = runner.state.events.last().expect("a park event");
18808        assert!(last.message.contains("0.121.0"), "{}", last.message);
18809    }
18810
18811    #[tokio::test]
18812    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18813        crate::run::pin_test_home();
18814        let mut runner = runner_at(RunStatus::Reviewing);
18815        runner.state.save().unwrap();
18816        let mut execution = runner.claim_driver().await.expect("first execution");
18817        assert_eq!(execution.runner.state.passes.len(), 1);
18818        execution
18819            .driver
18820            .finish(&mut execution.runner.state)
18821            .await
18822            .unwrap();
18823    }
18824
18825    #[tokio::test]
18826    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18827        crate::run::pin_test_home();
18828        let mut runner = runner_at(RunStatus::Reviewing);
18829        runner.state.config.agents = vec![conductor()];
18830        runner.state.config.graph.timeout_resource_wait = 0;
18831        runner.state.save().unwrap();
18832        let id = runner.state.id.clone();
18833        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18834        assert!(runner.claim_driver().await.is_err());
18835        drop(owner);
18836        drop(runner);
18837        let mut resumed = Runner::resume(&id).unwrap();
18838        assert_eq!(resumed.state.driver_claimed, Some(false));
18839        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18840        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18841        let mut execution = resumed
18842            .claim_driver()
18843            .await
18844            .expect("retry initial claim after reload");
18845        assert_eq!(execution.runner.state.driver_claimed, Some(true));
18846        assert_eq!(execution.runner.state.passes.len(), 1);
18847        execution
18848            .driver
18849            .finish(&mut execution.runner.state)
18850            .await
18851            .unwrap();
18852    }
18853
18854    #[tokio::test]
18855    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18856        crate::run::pin_test_home();
18857        let mut runner = runner_at(RunStatus::Reviewing);
18858        runner.state.config.agents = vec![conductor()];
18859        runner.state.save().unwrap();
18860        let id = runner.state.id.clone();
18861        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18862        let mut resumed = Runner::resume(&id).unwrap();
18863        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18864        assert!(resumed.claim_driver().await.is_err());
18865        assert!(RunState::load(&id).unwrap().passes.is_empty());
18866        drop(lease);
18867        let mut execution = resumed.claim_driver().await.unwrap();
18868        execution
18869            .driver
18870            .finish(&mut execution.runner.state)
18871            .await
18872            .unwrap();
18873    }
18874
18875    #[tokio::test]
18876    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18877        crate::run::pin_test_home();
18878        let mut runner = runner_at(RunStatus::Reviewing);
18879        let state = &mut runner.state;
18880        state.config.agents = vec![conductor()];
18881        state.driver_claimed = None;
18882        state.driver_pid = Some(std::process::id());
18883        state.driver_started_at = None;
18884        state.begin_pass();
18885        state.save().unwrap();
18886        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18887        std::fs::write(
18888            state.dir().join("driver.json"),
18889            serde_json::to_vec(&serde_json::json!({
18890                "pid": state.driver_pid, "started_at": null,
18891                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18892            }))
18893            .unwrap(),
18894        )
18895        .unwrap();
18896        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18897        drop(lease);
18898        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18899        let mut resumed = Runner::resume(&state.id).unwrap();
18900        assert!(!resumed.state.driver_exited);
18901        let mut execution = resumed
18902            .claim_driver()
18903            .await
18904            .expect("lease proves previous execution ended");
18905        assert_eq!(execution.runner.state.passes.len(), 2);
18906        execution
18907            .driver
18908            .finish(&mut execution.runner.state)
18909            .await
18910            .unwrap();
18911    }
18912
18913    #[tokio::test]
18914    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18915        crate::run::pin_test_home();
18916        let mut runner = runner_at(RunStatus::Reviewing);
18917        let mut json = serde_json::to_value(&runner.state).unwrap();
18918        json.as_object_mut().unwrap().remove("driver_claimed");
18919        runner.state = serde_json::from_value(json).unwrap();
18920        assert_eq!(runner.state.driver_claimed, None);
18921        runner.state.save().unwrap();
18922        let mut execution = runner
18923            .claim_driver()
18924            .await
18925            .expect("the execution lease, not a missing marker, excludes a second driver");
18926        assert_eq!(execution.runner.state.passes.len(), 1);
18927        execution
18928            .driver
18929            .finish(&mut execution.runner.state)
18930            .await
18931            .unwrap();
18932    }
18933
18934    #[tokio::test]
18935    async fn driver_admission_refuses_a_proven_live_driver() {
18936        crate::run::pin_test_home();
18937        let mut runner = runner_at(RunStatus::Reviewing);
18938        let pid = std::process::id();
18939        runner.state.driver_pid = Some(pid);
18940        runner.state.driver_started_at = crate::proc::process_started_at(pid);
18941        assert!(runner.state.driver_started_at.is_some());
18942        runner.state.save().unwrap();
18943        let error = runner
18944            .claim_driver()
18945            .await
18946            .err()
18947            .expect("a live driver must refuse");
18948        assert!(error.to_string().contains("live driver"), "{error:#}");
18949        let saved = RunState::load(&runner.state.id).unwrap();
18950        assert_eq!(saved.driver_pid, Some(pid));
18951        assert_eq!(saved.status, RunStatus::Reviewing);
18952        assert!(saved.passes.is_empty());
18953    }
18954
18955    #[test]
18956    fn parking_resource_halt_persists_the_pass_once() {
18957        crate::run::pin_test_home();
18958        let mut runner = runner_at(RunStatus::Reviewing);
18959        runner.state.begin_pass();
18960        runner.state.resource_block = Some(crate::run::ResourceBlock {
18961            node: "fix".into(),
18962            kind: "shared build cache".into(),
18963            detail: "cancelled".into(),
18964            at: Timestamp::now(),
18965            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18966        });
18967        let cause = ParkCause::Upgrade {
18968            from: "old".into(),
18969            to: Some("new".into()),
18970        };
18971        runner.pause.park_for(cause.clone());
18972        // Both direct fixer exits and graph resource boundaries must complete
18973        // parking, rather than returning a stale Reviewing record.
18974        runner.stop_for_resource().unwrap();
18975        assert!(runner.resource_halt().unwrap());
18976        assert!(runner.park_here().unwrap());
18977        let saved = RunState::load(&runner.state.id).unwrap();
18978        assert!(saved.parked);
18979        assert_eq!(saved.status, RunStatus::Reviewing);
18980        assert!(saved.passes[0].ended_at.is_some());
18981        assert_eq!(
18982            saved.passes[0].end,
18983            Some(crate::run::PassEnd::Parked {
18984                cause: cause.clone()
18985            })
18986        );
18987        assert_eq!(
18988            saved.resource_block.unwrap().cause,
18989            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18990        );
18991        assert_eq!(
18992            saved
18993                .events
18994                .iter()
18995                .filter(|event| event.node == "park")
18996                .count(),
18997            1
18998        );
18999    }
19000
19001    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
19002    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
19003    /// cleared) must not make a *different* run - one only watching its own,
19004    /// unshared interrupt `Pause` - see itself as parked. If a future change
19005    /// ever collapsed these back into one handle, the interrupt scheduler
19006    /// would park every run for the rest of the daemon's life, not just the
19007    /// one it meant to interrupt.
19008    #[test]
19009    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
19010        crate::run::pin_test_home();
19011        let mut runner = runner_at(RunStatus::Implementing);
19012        let shutdown = Pause::new();
19013        runner.on_pause(shutdown.clone());
19014        let interrupt = Pause::new();
19015        runner.watch_interrupt(interrupt.clone());
19016
19017        // Nobody has asked for anything yet.
19018        assert!(!runner.park_here().expect("park_here"));
19019        assert!(!runner.state.parked);
19020
19021        // Only the interrupt handle fires; the shutdown handle stays clear.
19022        interrupt.park_because("test");
19023        assert!(!shutdown.parked());
19024        assert!(runner.park_here().expect("park_here"));
19025    }
19026
19027    /// The property every prior attempt at this feature failed to pin down:
19028    /// asking a run to park while one of its nodes has a real, in-flight
19029    /// async operation running (an agent call, in production) must not cut
19030    /// that operation short. `park_here` is only ever consulted *between*
19031    /// `execute`'s node calls - see its own doc - so nothing inside a node
19032    /// can observe a park request until the node itself returns. This proves
19033    /// that structurally, with real `tokio` concurrency and a channel
19034    /// handshake (never a sleep, which would only prove "usually", not
19035    /// "cannot"): the "node" below reports that it has genuinely started,
19036    /// and only then is the park requested; the node still has to be told to
19037    /// finish before `park_here` is ever called, exactly mirroring every
19038    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
19039    /// in `execute`.
19040    #[tokio::test]
19041    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19042        crate::run::pin_test_home();
19043        let mut runner = runner_at(RunStatus::Implementing);
19044        let interrupt = Pause::new();
19045        runner.watch_interrupt(interrupt.clone());
19046
19047        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19048        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19049
19050        // Stands in for one node's in-flight agent call: it proves it has
19051        // genuinely started, then blocks - exactly as a spawned CLI process
19052        // does - until told to finish.
19053        let node = async move {
19054            started_tx.send(()).expect("send started");
19055            finish_rx.await.expect("recv finish");
19056            "node finished"
19057        };
19058
19059        let interrupter = async move {
19060            started_rx.await.expect("recv started");
19061            // The call is now genuinely in flight. Ask it to park.
19062            interrupt.park_because("higher-priority task waiting");
19063            // Nothing the node does can observe this yet - there is no
19064            // check inside it, by construction - so let the executor run
19065            // anything pending and then let the node finish on its own.
19066            tokio::task::yield_now().await;
19067            finish_tx.send(()).expect("send finish");
19068        };
19069
19070        let (node_result, ()) = tokio::join!(node, interrupter);
19071        assert_eq!(
19072            node_result, "node finished",
19073            "the in-flight call ran to completion"
19074        );
19075
19076        // Only now, at the boundary the real `execute` would check right
19077        // after this node, does the park take effect.
19078        assert!(runner.park_here().expect("park_here"));
19079        assert!(runner.state.parked);
19080    }
19081
19082    /// A run parked mid-competition carries every field it had accumulated
19083    /// through the exact same disk round-trip an ordinary resume uses -
19084    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
19085    /// Nothing about parking for an interrupt is a special case of that path;
19086    /// this is what proves it rather than assuming it.
19087    #[test]
19088    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19089        crate::run::pin_test_home();
19090        let mut runner = runner_at(RunStatus::Judging);
19091        // `Runner::resume` re-resolves roles from the saved config, which
19092        // refuses an empty roster - give it the same minimal one `conductor`
19093        // itself uses.
19094        runner.state.config.agents = vec![conductor()];
19095        runner.state.candidates = vec![Candidate {
19096            index: 0,
19097            label: 'A',
19098            agent: "alpha".to_owned(),
19099            branch: "magi/x/A".to_owned(),
19100            worktree: PathBuf::from("/nonexistent/worktree"),
19101            summary: "did the thing".to_owned(),
19102            stat: "1 file changed".to_owned(),
19103            files: 1,
19104            commits: 1,
19105            empty: false,
19106            failed: None,
19107            verified_noop: None,
19108            duration_ms: 1234,
19109            folded: false,
19110        }];
19111        let run_id = runner.state.id.clone();
19112
19113        let interrupt = Pause::new();
19114        runner.watch_interrupt(interrupt.clone());
19115        interrupt.park_because("task c3d4 asked to run first");
19116        assert!(runner.park_here().expect("park_here"));
19117
19118        let resumed = Runner::resume(&run_id).expect("resume");
19119        assert_eq!(resumed.state.candidates.len(), 1);
19120        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19121        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19122        assert_eq!(resumed.state.status, runner.state.status);
19123        assert!(
19124            resumed.state.parked,
19125            "still parked until `execute` actually walks the graph again"
19126        );
19127        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19128    }
19129
19130    /// A fresh open question on `run`, stored and handed back for assertions.
19131    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19132        let mut q = ask::Question::new(
19133            run.to_owned(),
19134            "implement".to_owned(),
19135            "impl-A".to_owned(),
19136            "Which storage backend should the cache use?".to_owned(),
19137            String::new(),
19138            vec!["SQLite".to_owned(), "Redis".to_owned()],
19139        );
19140        store.put(&mut q).unwrap();
19141        q
19142    }
19143
19144    #[test]
19145    fn a_failed_runs_open_question_is_abandoned() {
19146        ask_test_home();
19147        let store = ask::Questions::open();
19148        let mut runner = runner_at(RunStatus::Failed);
19149        let run = runner.state.id.clone();
19150        let q = ask_open_question(&store, &run);
19151
19152        runner.settle_questions();
19153
19154        let back = store.get(&q.id).unwrap();
19155        assert!(
19156            !back.status.open(),
19157            "the seat that asked died with the run; nobody is left to read an answer"
19158        );
19159        assert!(
19160            back.detail.contains(&run) && back.detail.contains("failed"),
19161            "the reason names what the run became, not just that it is gone: {}",
19162            back.detail
19163        );
19164    }
19165
19166    #[test]
19167    fn a_merged_runs_open_question_is_abandoned_too() {
19168        ask_test_home();
19169        let store = ask::Questions::open();
19170        // A run that finishes cleanly still leaves nobody to read an answer -
19171        // this is not only a failure-path cleanup.
19172        for status in [RunStatus::Merged, RunStatus::Ready] {
19173            let mut runner = runner_at(status);
19174            let run = runner.state.id.clone();
19175            let q = ask_open_question(&store, &run);
19176
19177            runner.settle_questions();
19178
19179            let back = store.get(&q.id).unwrap();
19180            assert!(
19181                !back.status.open(),
19182                "{status:?} run's question must not outlive the run"
19183            );
19184        }
19185    }
19186
19187    #[test]
19188    fn a_still_resumable_runs_open_question_is_left_alone() {
19189        ask_test_home();
19190        let store = ask::Questions::open();
19191        // `Blocked` and `Stalled` can still be resumed — the candidates, the
19192        // review round and the seat sessions are all still on disk — so a
19193        // question asked mid-round may yet get a real answer from a real
19194        // resume. Sweeping it here would be exactly the failure mode this
19195        // whole feature exists to avoid on the other side.
19196        for status in [RunStatus::Blocked, RunStatus::Stalled] {
19197            let mut runner = runner_at(status);
19198            let run = runner.state.id.clone();
19199            let q = ask_open_question(&store, &run);
19200
19201            runner.settle_questions();
19202
19203            let back = store.get(&q.id).unwrap();
19204            assert!(
19205                back.status.open(),
19206                "{status:?} is still alive; the question must still be waiting"
19207            );
19208        }
19209    }
19210
19211    #[test]
19212    fn settle_questions_never_touches_an_already_answered_question() {
19213        ask_test_home();
19214        let store = ask::Questions::open();
19215        let mut runner = runner_at(RunStatus::Failed);
19216        let run = runner.state.id.clone();
19217        let mut q = ask_open_question(&store, &run);
19218        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19219            .unwrap();
19220        store.put(&mut q).unwrap();
19221
19222        // Called twice, the way a crash-recovered daemon reclaim and the
19223        // graph's own cleanup both can for the same run — `abandon_for_run`
19224        // only ever touches what is still open, so this must be inert both
19225        // times, not merely the second.
19226        runner.settle_questions();
19227        runner.settle_questions();
19228
19229        let back = store.get(&q.id).unwrap();
19230        assert_eq!(
19231            back.status,
19232            ask::QuestionStatus::Answered,
19233            "a real answer is a decision on record, never overwritten by a sweep"
19234        );
19235    }
19236
19237    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
19238    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
19239    /// without merging, whose winner is still the operator's answer to read.
19240    /// Nothing previously called `fold_run` itself with a real `tally`, so
19241    /// this is the first test to pin down the one distinction the whole
19242    /// automatic-fold feature depends on: the winner's worktree and branch
19243    /// must survive, everything else sharing the run's worktree bay - a
19244    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
19245    /// second sweep treats every non-winner directory under the bay alike -
19246    /// must not.
19247    #[tokio::test]
19248    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19249        crate::run::pin_test_home();
19250        let tmp = tempfile::tempdir().expect("tempdir");
19251        let repo = tmp.path().join("repo");
19252        std::fs::create_dir_all(&repo).unwrap();
19253        init_repo(&repo);
19254
19255        let mut config = Config::default();
19256        config.graph.worktree_root = Some(tmp.path().join("wt"));
19257
19258        let mut state = RunState::new(
19259            repo.clone(),
19260            "main".to_owned(),
19261            "deadbeef".to_owned(),
19262            "task".to_owned(),
19263            config,
19264        );
19265        let root = state.worktree_root();
19266        let wt_a = root.join("cand-A");
19267        let wt_b = root.join("cand-B");
19268        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19269            .await
19270            .expect("worktree A");
19271        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19272            .await
19273            .expect("worktree B");
19274
19275        state.candidates = vec![
19276            Candidate {
19277                index: 0,
19278                label: 'A',
19279                agent: "alpha".to_owned(),
19280                branch: "magi/x/A".to_owned(),
19281                worktree: wt_a.clone(),
19282                summary: String::new(),
19283                stat: String::new(),
19284                files: 0,
19285                commits: 0,
19286                empty: false,
19287                failed: None,
19288                verified_noop: None,
19289                duration_ms: 0,
19290                folded: false,
19291            },
19292            Candidate {
19293                index: 1,
19294                label: 'B',
19295                agent: "beta".to_owned(),
19296                branch: "magi/x/B".to_owned(),
19297                worktree: wt_b.clone(),
19298                summary: String::new(),
19299                stat: String::new(),
19300                files: 0,
19301                commits: 0,
19302                empty: false,
19303                failed: None,
19304                verified_noop: None,
19305                duration_ms: 0,
19306                folded: false,
19307            },
19308        ];
19309        state.tally = Some(Tally {
19310            first_choice: BTreeMap::from([('A', 1)]),
19311            borda: BTreeMap::new(),
19312            winner: 'A',
19313            rankings: 1,
19314            unanimous_initial: true,
19315            deliberated: false,
19316            changed_votes: 0,
19317            unanimous_final: true,
19318            tie_break: None,
19319            judges: 1,
19320            present: 1,
19321            quorum: 1,
19322            met_quorum: true,
19323            uncontested: None,
19324        });
19325        state.status = RunStatus::Ready;
19326
19327        fold_run(&mut state, false, &crate::run::home())
19328            .await
19329            .expect("fold_run");
19330
19331        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19332        assert!(
19333            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19334            "the unmerged winner's branch survives"
19335        );
19336        assert!(
19337            !state.candidates[0].folded,
19338            "the winner is not marked folded"
19339        );
19340
19341        assert!(!wt_b.exists(), "the loser's worktree is removed");
19342        assert!(
19343            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19344            "the loser's branch is removed"
19345        );
19346        assert!(state.candidates[1].folded, "the loser is marked folded");
19347    }
19348
19349    #[tokio::test]
19350    #[cfg(unix)]
19351    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19352        use std::os::fd::AsRawFd as _;
19353        use std::time::Duration;
19354
19355        let tmp = tempfile::tempdir().unwrap();
19356        let repo = tmp.path().join("repo");
19357        std::fs::create_dir_all(&repo).unwrap();
19358        init_repo(&repo);
19359        let common = repo.join(".git");
19360        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19361        let home = tmp.path().join("home");
19362        let mut config = Config::default();
19363        config.coordination = crate::config::test_coordination();
19364        config.graph.worktree_root = Some(tmp.path().join("wt"));
19365        config.graph.timeout_resource_wait = 1;
19366        let mut state = RunState::new(
19367            repo.clone(),
19368            "main".into(),
19369            "deadbeef".into(),
19370            "task".into(),
19371            config,
19372        );
19373        let root = state.worktree_root();
19374        let candidate = root.join("cand-A");
19375        let reviewer = root.join("review-1");
19376        let branch = state.branch_for('A');
19377        git::worktree_add_branch(&repo, &candidate, &branch, "main")
19378            .await
19379            .unwrap();
19380        git::worktree_add_detached(&repo, &reviewer, "main")
19381            .await
19382            .unwrap();
19383        git::acquire_worktree_config(&repo).await.unwrap();
19384        state.enabled_worktree_config = true;
19385        state.status = RunStatus::Ready;
19386        state.candidates.push(Candidate {
19387            index: 0,
19388            label: 'A',
19389            agent: "implement".into(),
19390            branch: branch.clone(),
19391            worktree: candidate.clone(),
19392            summary: String::new(),
19393            stat: String::new(),
19394            files: 0,
19395            commits: 0,
19396            empty: false,
19397            failed: None,
19398            verified_noop: None,
19399            duration_ms: 0,
19400            folded: false,
19401        });
19402        state.save_under(&home).unwrap();
19403        let config_before = std::fs::read(common.join("config")).unwrap();
19404        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19405            .await
19406            .unwrap();
19407        // A separate OS handle bypasses the launcher's local gate, exactly as
19408        // an external updater does, and must still prevent all folding work.
19409        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19410        assert_eq!(
19411            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19412            0
19413        );
19414        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19415        assert!(
19416            tokio::time::timeout(Duration::from_millis(200), &mut fold)
19417                .await
19418                .is_err()
19419        );
19420        assert!(candidate.exists());
19421        assert!(reviewer.exists());
19422        assert!(git::branch_exists(&repo, &branch).await.unwrap());
19423        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19424        assert_eq!(
19425            git::git(&repo, &["worktree", "list", "--porcelain"])
19426                .await
19427                .unwrap(),
19428            worktrees_before
19429        );
19430        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19431            .await
19432            .unwrap()
19433            .unwrap_err();
19434        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19435        drop(fold);
19436        assert!(candidate.exists() && reviewer.exists());
19437        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19438        assert_eq!(
19439            state.resource_block.as_ref().unwrap().kind,
19440            "repository coordination"
19441        );
19442        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19443        drop(external);
19444        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19445        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19446            .await
19447            .unwrap()
19448            .unwrap();
19449        drop(fold);
19450        assert!(removed.contains(&branch));
19451        assert!(!root.exists());
19452        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19453        assert!(state.candidates[0].folded);
19454        assert!(!state.enabled_worktree_config);
19455        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19456        let enabled = git::git_raw(
19457            &repo,
19458            &["config", "--local", "--get", "extensions.worktreeConfig"],
19459        )
19460        .await
19461        .unwrap();
19462        assert!(!enabled.ok(), "the last config reference was released");
19463    }
19464
19465    /// A branch handed to a later run is that run's (and its pull request's):
19466    /// folding the run that released it must not delete it.
19467    #[tokio::test]
19468    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19469        let tmp = tempfile::tempdir().expect("tempdir");
19470        let repo = tmp.path().join("repo");
19471        std::fs::create_dir_all(&repo).unwrap();
19472        init_repo(&repo);
19473        let home = tmp.path().join("home");
19474
19475        let mut config = Config::default();
19476        config.graph.worktree_root = Some(tmp.path().join("wt"));
19477        let mut state = RunState::new(
19478            repo.clone(),
19479            "main".to_owned(),
19480            "deadbeef".to_owned(),
19481            "task".to_owned(),
19482            config,
19483        );
19484        // The worktree is already gone (released); the branch survives.
19485        git::git(&repo, &["branch", "magi/x/A", "main"])
19486            .await
19487            .expect("branch");
19488        state.candidates = vec![Candidate {
19489            index: 0,
19490            label: 'A',
19491            agent: "alpha".to_owned(),
19492            branch: "magi/x/A".to_owned(),
19493            worktree: state.worktree_root().join("cand-A"),
19494            summary: String::new(),
19495            stat: String::new(),
19496            files: 0,
19497            commits: 0,
19498            empty: false,
19499            failed: None,
19500            verified_noop: None,
19501            duration_ms: 0,
19502            folded: true,
19503        }];
19504        state.released_to = Some("20260901-000000-new1".to_owned());
19505        state.released_branches = vec!["magi/x/A".to_owned()];
19506
19507        fold_run(&mut state, true, &home).await.expect("fold_run");
19508        state.status = RunStatus::Merged;
19509        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19510        state.merge = Some(crate::run::MergeOutcome {
19511            mode: MergeMode::Pr,
19512            ok: true,
19513            detail: "confirmed merged".into(),
19514            empty: false,
19515        });
19516        fold_run(&mut state, true, &home)
19517            .await
19518            .expect("merged fold");
19519
19520        assert!(
19521            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19522            "the handed-over branch survives a fold"
19523        );
19524    }
19525
19526    /// A winner with nothing ahead of the base is caught before `gh` is ever
19527    /// asked for a pull request, and the message carries what the task's
19528    /// references resolved to.
19529    #[tokio::test]
19530    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19531        let tmp = tempfile::tempdir().expect("tempdir");
19532        let repo = tmp.path().join("repo");
19533        std::fs::create_dir_all(&repo).unwrap();
19534        init_repo(&repo);
19535        let run = |args: &[&str]| {
19536            let out = std::process::Command::new("git")
19537                .quiet()
19538                .args(args)
19539                .current_dir(&repo)
19540                .output()
19541                .expect("spawn git");
19542            assert!(out.status.success(), "git {args:?}");
19543        };
19544        run(&["branch", "magi/x/A"]);
19545        run(&["checkout", "-q", "-b", "magi/x/B"]);
19546        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19547        run(&["add", "-A"]);
19548        run(&["commit", "-q", "-m", "work"]);
19549        run(&["checkout", "-q", "main"]);
19550
19551        // A pull request is compared against the remote's base, so the
19552        // fixture needs one. Before it exists nothing can be read, and the
19553        // answer must be "not empty".
19554        let probe = RunState::new(
19555            repo.clone(),
19556            "main".to_owned(),
19557            "deadbeef".to_owned(),
19558            "task".to_owned(),
19559            Config::default(),
19560        );
19561        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19562        let bare = tmp.path().join("origin.git");
19563        let out = std::process::Command::new("git")
19564            .quiet()
19565            .args(["init", "-q", "--bare"])
19566            .arg(&bare)
19567            .output()
19568            .expect("spawn git");
19569        assert!(out.status.success(), "git init --bare");
19570        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19571        run(&["push", "-q", "origin", "main"]);
19572
19573        let mut state = RunState::new(
19574            repo.clone(),
19575            "main".to_owned(),
19576            "deadbeef".to_owned(),
19577            "task".to_owned(),
19578            Config::default(),
19579        );
19580        state.seeds = vec![refs::Seed {
19581            token: "magi/27b2/A".to_owned(),
19582            kind: refs::SeedKind::Unresolved,
19583            sha: String::new(),
19584            branch: true,
19585            detail: "no branch or commit named magi/27b2/A".to_owned(),
19586        }];
19587
19588        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19589        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19590        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19591        let detail = empty_candidate_detail(&state, "main");
19592        assert!(detail.starts_with("empty candidate"), "{detail}");
19593        assert!(!detail.contains("magi/27b2/A"), "{detail}");
19594        assert!(detail.contains("--start-from"), "{detail}");
19595    }
19596
19597    /// `status == Ready` used to be read as "this is the harmless
19598    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
19599    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
19600    /// run's PR was closed without merging — and reentering `merge` with
19601    /// `mode` still `Pr` does not know the difference, so it pushed and
19602    /// opened a second pull request. `mode == Local` reproduces the same
19603    /// blind spot without a network call: reentry must not attempt another
19604    /// git merge once this node has already recorded an outcome.
19605    #[tokio::test]
19606    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19607        let tmp = tempfile::tempdir().expect("tempdir");
19608        let repo = tmp.path().join("repo");
19609        std::fs::create_dir_all(&repo).unwrap();
19610        init_repo(&repo);
19611
19612        let mut config = Config::default();
19613        config.merge.mode = MergeMode::Local;
19614
19615        let mut state = RunState::new(
19616            repo.clone(),
19617            "main".to_owned(),
19618            "deadbeef".to_owned(),
19619            "task".to_owned(),
19620            config,
19621        );
19622        state.candidates = vec![Candidate {
19623            index: 0,
19624            label: 'A',
19625            agent: "alpha".to_owned(),
19626            branch: "does-not-exist".to_owned(),
19627            worktree: repo.clone(),
19628            summary: String::new(),
19629            stat: String::new(),
19630            files: 0,
19631            commits: 0,
19632            empty: false,
19633            failed: None,
19634            verified_noop: None,
19635            duration_ms: 0,
19636            folded: false,
19637        }];
19638        state.tally = Some(Tally {
19639            first_choice: BTreeMap::from([('A', 1)]),
19640            borda: BTreeMap::new(),
19641            winner: 'A',
19642            rankings: 1,
19643            unanimous_initial: true,
19644            deliberated: false,
19645            changed_votes: 0,
19646            unanimous_final: true,
19647            tie_break: None,
19648            judges: 0,
19649            present: 0,
19650            quorum: 0,
19651            met_quorum: true,
19652            uncontested: Some("only candidate A produced a change".to_owned()),
19653        });
19654        state.reviews = vec![ReviewRound {
19655            round: 1,
19656            head: "deadbeef".to_owned(),
19657            verified_head: None,
19658            verified_at: None,
19659            reviews: Vec::new(),
19660            e2e: Vec::new(),
19661            fix: None,
19662            blocking: 0,
19663            answered: 0,
19664            expected: 0,
19665            clean: true,
19666            verify_retried: false,
19667            e2e_deferred: false,
19668            e2e_defer_reason: None,
19669            progressed: false,
19670            vote_split: false,
19671            reconsideration: Vec::new(),
19672            verdict: None,
19673        }];
19674        state.gate = vec![CommandOutcome {
19675            command: "test".to_owned(),
19676            code: Some(0),
19677            output_tail: String::new(),
19678            duration_ms: 0,
19679            resource_blocked: false,
19680        }];
19681        state.gate_ran = true;
19682        // Reached its conclusion already — e.g. `land` closing the PR without
19683        // merging it, which (like the honest `MergeMode::None` path) leaves
19684        // `status` at `Ready`. The recorded outcome is what actually marks
19685        // this node done.
19686        state.status = RunStatus::Ready;
19687        state.merge = Some(MergeOutcome {
19688            mode: MergeMode::Local,
19689            ok: false,
19690            detail: "already concluded".to_owned(),
19691            empty: false,
19692        });
19693
19694        let mut runner = Runner {
19695            state,
19696            roles: ResolvedRoles {
19697                implementers: Vec::new(),
19698                judges: Vec::new(),
19699                reviewers: Vec::new(),
19700                fixer: None,
19701                conductor: conductor(),
19702                implementer_roster: Vec::new(),
19703                judge_roster: Vec::new(),
19704                reviewer_roster: Vec::new(),
19705            },
19706            sem: Arc::new(Semaphore::new(1)),
19707            pause: Pause::new(),
19708            interrupt: Pause::new(),
19709        };
19710
19711        runner.merge().await.expect("merge");
19712
19713        assert_eq!(
19714            runner.state.status,
19715            RunStatus::Ready,
19716            "a concluded run's status must not change on reentry"
19717        );
19718        assert_eq!(
19719            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19720            Some("already concluded"),
19721            "merge must not run again once the node already recorded an outcome"
19722        );
19723    }
19724
19725    /// `gate` leaves `state.gate_ran` false both before it has ever run and
19726    /// when its last attempt was resource-blocked (the shared build cache
19727    /// could not be acquired or confirmed fresh in time - see
19728    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
19729    /// `Vec` this also leaves behind used to read as "nothing failed" and let
19730    /// a run merge a tree the gate never actually checked - exactly the case
19731    /// a contended cache produces on every retry until it clears. `merge`
19732    /// must refuse until `gate` has actually recorded an attempt.
19733    #[tokio::test]
19734    async fn merge_refuses_a_gate_that_has_not_actually_run() {
19735        let tmp = tempfile::tempdir().expect("tempdir");
19736        let repo = tmp.path().join("repo");
19737        std::fs::create_dir_all(&repo).unwrap();
19738        init_repo(&repo);
19739
19740        let mut config = Config::default();
19741        config.merge.mode = MergeMode::Local;
19742
19743        let mut state = RunState::new(
19744            repo.clone(),
19745            "main".to_owned(),
19746            "deadbeef".to_owned(),
19747            "task".to_owned(),
19748            config,
19749        );
19750        state.candidates = vec![Candidate {
19751            index: 0,
19752            label: 'A',
19753            agent: "alpha".to_owned(),
19754            branch: "does-not-exist".to_owned(),
19755            worktree: repo.clone(),
19756            summary: String::new(),
19757            stat: String::new(),
19758            files: 0,
19759            commits: 0,
19760            empty: false,
19761            failed: None,
19762            verified_noop: None,
19763            duration_ms: 0,
19764            folded: false,
19765        }];
19766        state.tally = Some(Tally {
19767            first_choice: BTreeMap::from([('A', 1)]),
19768            borda: BTreeMap::new(),
19769            winner: 'A',
19770            rankings: 1,
19771            unanimous_initial: true,
19772            deliberated: false,
19773            changed_votes: 0,
19774            unanimous_final: true,
19775            tie_break: None,
19776            judges: 0,
19777            present: 0,
19778            quorum: 0,
19779            met_quorum: true,
19780            uncontested: Some("only candidate A produced a change".to_owned()),
19781        });
19782        state.reviews = vec![ReviewRound {
19783            round: 1,
19784            head: "deadbeef".to_owned(),
19785            verified_head: None,
19786            verified_at: None,
19787            reviews: Vec::new(),
19788            e2e: Vec::new(),
19789            fix: None,
19790            blocking: 0,
19791            answered: 0,
19792            expected: 0,
19793            clean: true,
19794            verify_retried: false,
19795            e2e_deferred: false,
19796            e2e_defer_reason: None,
19797            progressed: false,
19798            vote_split: false,
19799            reconsideration: Vec::new(),
19800            verdict: None,
19801        }];
19802        // The point: `gate` has not recorded anything yet.
19803        state.gate = Vec::new();
19804        state.gate_ran = false;
19805        state.status = RunStatus::Gating;
19806
19807        let mut runner = Runner {
19808            state,
19809            roles: ResolvedRoles {
19810                implementers: Vec::new(),
19811                judges: Vec::new(),
19812                reviewers: Vec::new(),
19813                fixer: None,
19814                conductor: conductor(),
19815                implementer_roster: Vec::new(),
19816                judge_roster: Vec::new(),
19817                reviewer_roster: Vec::new(),
19818            },
19819            sem: Arc::new(Semaphore::new(1)),
19820            pause: Pause::new(),
19821            interrupt: Pause::new(),
19822        };
19823
19824        runner.merge().await.expect("merge");
19825
19826        assert!(
19827            runner.state.merge.is_none(),
19828            "an empty gate must never be read as a passing one: {:?}",
19829            runner.state.merge
19830        );
19831    }
19832
19833    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
19834    /// commands configured and `merge.mode` is `none` (a review-only run).
19835    /// `gate` must still record a real attempt — zero commands, vacuously
19836    /// passed — rather than leaving `state.gate` empty in a way `merge`
19837    /// cannot tell apart from "never ran"; otherwise the run reaches
19838    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
19839    #[tokio::test]
19840    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19841        let tmp = tempfile::tempdir().expect("tempdir");
19842        let repo = tmp.path().join("repo");
19843        std::fs::create_dir_all(&repo).unwrap();
19844        init_repo(&repo);
19845
19846        // Default config: `verify.gate` empty, `merge.mode` is `none`.
19847        let config = Config::default();
19848
19849        let mut state = RunState::new(
19850            repo.clone(),
19851            "main".to_owned(),
19852            "deadbeef".to_owned(),
19853            "task".to_owned(),
19854            config,
19855        );
19856        state.candidates = vec![Candidate {
19857            index: 0,
19858            label: 'A',
19859            agent: "alpha".to_owned(),
19860            branch: "does-not-exist".to_owned(),
19861            worktree: repo.clone(),
19862            summary: String::new(),
19863            stat: String::new(),
19864            files: 0,
19865            commits: 0,
19866            empty: false,
19867            failed: None,
19868            verified_noop: None,
19869            duration_ms: 0,
19870            folded: false,
19871        }];
19872        state.tally = Some(Tally {
19873            first_choice: BTreeMap::from([('A', 1)]),
19874            borda: BTreeMap::new(),
19875            winner: 'A',
19876            rankings: 1,
19877            unanimous_initial: true,
19878            deliberated: false,
19879            changed_votes: 0,
19880            unanimous_final: true,
19881            tie_break: None,
19882            judges: 0,
19883            present: 0,
19884            quorum: 0,
19885            met_quorum: true,
19886            uncontested: Some("only candidate A produced a change".to_owned()),
19887        });
19888        state.reviews = vec![ReviewRound {
19889            round: 1,
19890            head: "deadbeef".to_owned(),
19891            verified_head: None,
19892            verified_at: None,
19893            reviews: Vec::new(),
19894            e2e: Vec::new(),
19895            fix: None,
19896            blocking: 0,
19897            answered: 0,
19898            expected: 0,
19899            clean: true,
19900            verify_retried: false,
19901            e2e_deferred: false,
19902            e2e_defer_reason: None,
19903            progressed: false,
19904            vote_split: false,
19905            reconsideration: Vec::new(),
19906            verdict: None,
19907        }];
19908
19909        let mut runner = Runner {
19910            state,
19911            roles: ResolvedRoles {
19912                implementers: Vec::new(),
19913                judges: Vec::new(),
19914                reviewers: Vec::new(),
19915                fixer: None,
19916                conductor: conductor(),
19917                implementer_roster: Vec::new(),
19918                judge_roster: Vec::new(),
19919                reviewer_roster: Vec::new(),
19920            },
19921            sem: Arc::new(Semaphore::new(1)),
19922            pause: Pause::new(),
19923            interrupt: Pause::new(),
19924        };
19925
19926        runner.gate().await.expect("gate");
19927        assert!(
19928            runner.state.gate_ran,
19929            "zero configured commands is still a real attempt, not an unrun gate"
19930        );
19931        assert!(runner.state.gate.is_empty());
19932        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19933        assert_ne!(
19934            runner.state.status,
19935            RunStatus::Blocked,
19936            "a gate with nothing to check must not read as failed"
19937        );
19938
19939        runner.merge().await.expect("merge");
19940        assert_eq!(
19941            runner.state.status,
19942            RunStatus::Ready,
19943            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19944        );
19945    }
19946
19947    /// `Config::cache_dir` is derived from `verify.e2e` as well as
19948    /// `verify.gate` (so the e2e leg and the final gate never build against
19949    /// different directories). With zero `verify.gate` commands but a
19950    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
19951    /// that lease before discovering it had nothing to run - so a repo with
19952    /// no gate commands could come back `resource_blocked` (and therefore
19953    /// still `gate_ran == false`) on nothing but an unrelated run holding the
19954    /// cache, exactly the contention this run's own zero commands could
19955    /// never have touched. `gate` must recognise there is nothing to check
19956    /// before it ever asks for the lease.
19957    #[tokio::test]
19958    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19959        crate::run::pin_test_home();
19960        let home = crate::run::home();
19961
19962        let tmp = tempfile::tempdir().expect("tempdir");
19963        let repo = tmp.path().join("repo");
19964        std::fs::create_dir_all(&repo).unwrap();
19965        init_repo(&repo);
19966        // Unique to this test, so holding its lease cannot collide with
19967        // another test sharing the same process-wide `home`.
19968        let cache_dir = tmp.path().join("target");
19969
19970        let mut config = Config::default();
19971        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19972        // `verify.gate` stays empty (the default). Any accidental cache
19973        // acquisition must report contention immediately, regardless of how
19974        // long unrelated Git subprocesses take under parallel test load.
19975        config.graph.timeout_verify = Some(2);
19976        config.graph.timeout_resource_wait = 0;
19977
19978        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19979        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19980            .expect("no io error acquiring directly")
19981        {
19982            crate::cache::AcquireOutcome::Acquired(g) => g,
19983            crate::cache::AcquireOutcome::Busy(b) => {
19984                panic!("expected the direct acquire to win the lease first: {b:?}")
19985            }
19986        };
19987
19988        let mut state = RunState::new(
19989            repo.clone(),
19990            "main".to_owned(),
19991            "deadbeef".to_owned(),
19992            "task".to_owned(),
19993            config,
19994        );
19995        state.candidates = vec![Candidate {
19996            index: 0,
19997            label: 'A',
19998            agent: "alpha".to_owned(),
19999            branch: "does-not-exist".to_owned(),
20000            worktree: repo.clone(),
20001            summary: String::new(),
20002            stat: String::new(),
20003            files: 0,
20004            commits: 0,
20005            empty: false,
20006            failed: None,
20007            verified_noop: None,
20008            duration_ms: 0,
20009            folded: false,
20010        }];
20011        state.tally = Some(Tally {
20012            first_choice: BTreeMap::from([('A', 1)]),
20013            borda: BTreeMap::new(),
20014            winner: 'A',
20015            rankings: 1,
20016            unanimous_initial: true,
20017            deliberated: false,
20018            changed_votes: 0,
20019            unanimous_final: true,
20020            tie_break: None,
20021            judges: 0,
20022            present: 0,
20023            quorum: 0,
20024            met_quorum: true,
20025            uncontested: Some("only candidate A produced a change".to_owned()),
20026        });
20027        state.reviews = vec![ReviewRound {
20028            round: 1,
20029            head: "deadbeef".to_owned(),
20030            verified_head: None,
20031            verified_at: None,
20032            reviews: Vec::new(),
20033            e2e: Vec::new(),
20034            fix: None,
20035            blocking: 0,
20036            answered: 0,
20037            expected: 0,
20038            clean: true,
20039            verify_retried: false,
20040            e2e_deferred: false,
20041            e2e_defer_reason: None,
20042            progressed: false,
20043            vote_split: false,
20044            reconsideration: Vec::new(),
20045            verdict: None,
20046        }];
20047
20048        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20049        state.reviews[0].e2e = vec![CommandOutcome {
20050            command: state.config.verify.e2e[0].clone(),
20051            code: Some(0),
20052            output_tail: String::new(),
20053            duration_ms: 0,
20054            resource_blocked: false,
20055        }];
20056        let mut runner = Runner {
20057            state,
20058            roles: ResolvedRoles {
20059                implementers: Vec::new(),
20060                judges: Vec::new(),
20061                reviewers: Vec::new(),
20062                fixer: None,
20063                conductor: conductor(),
20064                implementer_roster: Vec::new(),
20065                judge_roster: Vec::new(),
20066                reviewer_roster: Vec::new(),
20067            },
20068            sem: Arc::new(Semaphore::new(1)),
20069            pause: Pause::new(),
20070            interrupt: Pause::new(),
20071        };
20072
20073        runner.gate().await.expect("gate");
20074        assert!(runner.state.resource_block.is_none());
20075        assert!(
20076            !runner
20077                .state
20078                .events
20079                .iter()
20080                .any(|event| { event.message.contains("waiting for the shared build cache") }),
20081            "a gate with nothing to run must never request the held cache lease"
20082        );
20083        assert!(
20084            runner.state.gate_ran,
20085            "zero commands is still a real, immediate attempt"
20086        );
20087        assert!(runner.state.gate.is_empty());
20088        assert_ne!(
20089            runner.state.status,
20090            RunStatus::Blocked,
20091            "must not read as resource-blocked on a lease it never asked for"
20092        );
20093    }
20094
20095    /// A separate process carries a live worker's environment throughout all
20096    /// verification entry points; the parallel harness never changes its env.
20097    #[cfg(unix)]
20098    #[tokio::test]
20099    async fn verification_is_standalone_inside_a_live_worker() {
20100        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20101        if std::env::var_os(CHILD).is_none() {
20102            let home = tempfile::tempdir().unwrap();
20103            let generation = crate::worker::pin_current(home.path()).unwrap();
20104            let output = std::process::Command::new(std::env::current_exe().unwrap())
20105                .args([
20106                    "--exact",
20107                    "graph::tests::verification_is_standalone_inside_a_live_worker",
20108                    "--nocapture",
20109                ])
20110                .env(CHILD, "1")
20111                .env("MAGI_HOME", home.path())
20112                .env("MAGI_WORKER_HOME", home.path())
20113                .env("MAGI_WORKER_ID", "abcdef")
20114                .env("MAGI_WORKER_GENERATION", &generation.id)
20115                .env("MAGI_WORKER_EXE", &generation.executable)
20116                .env("MAGI_RUN", "verification-parent")
20117                .env("MAGI_NODE", "gate")
20118                .env("MAGI_SEAT", "worker-fixture")
20119                .env("MAGI_PROCESS_OWNER", "fixture-owner")
20120                .env("MAGI_NO_AUTOUPDATE", "0")
20121                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
20122                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
20123                .env(
20124                    "VERIFY_EXPECT_TOOLCHAIN",
20125                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
20126                )
20127                .env(
20128                    "VERIFY_EXPECT_CACHE",
20129                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
20130                )
20131                .output()
20132                .unwrap();
20133            assert!(
20134                output.status.success()
20135                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
20136                "{}\n{}",
20137                String::from_utf8_lossy(&output.stdout),
20138                String::from_utf8_lossy(&output.stderr)
20139            );
20140            return;
20141        }
20142        let home = crate::run::home();
20143        let directory = home.join("workers/jobs/abcdef");
20144        std::fs::create_dir_all(&directory).unwrap();
20145        let job = crate::worker::Job {
20146            id: "abcdef".into(),
20147            sequence: 1,
20148            kind: "task".into(),
20149            key: "task:verification".into(),
20150            payload: serde_json::json!({}),
20151            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
20152            version: Some(env!("CARGO_PKG_VERSION").into()),
20153            status: crate::worker::Status::Running,
20154            result: None,
20155            pid: Some(std::process::id()),
20156            created_at: 0,
20157            finished_at: None,
20158        };
20159        std::fs::write(
20160            directory.join("job.json"),
20161            serde_json::to_vec(&job).unwrap(),
20162        )
20163        .unwrap();
20164        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
20165        let assert_worker = || {
20166            assert_eq!(
20167                crate::worker::current_job(&home).unwrap().unwrap().id,
20168                job.id
20169            );
20170            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
20171            assert_eq!(
20172                crate::worker::helper_executable(&home).unwrap(),
20173                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20174            );
20175            assert!(
20176                crate::updater::Checker::new(&crate::config::Update {
20177                    mode: crate::config::UpdateMode::Notify,
20178                    interval: None,
20179                    auto_restart: false,
20180                })
20181                .is_none(),
20182                "the driver remains pinned, even while tests are standalone"
20183            );
20184        };
20185        assert_worker();
20186        let repo = home.join("repo");
20187        std::fs::create_dir_all(&repo).unwrap();
20188        init_repo(&repo);
20189        // Keep storage, seat/control context, containment and tool lookup.
20190        // No inherited values (or credentials) are printed on success.
20191        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20192            test -z "${MAGI_WORKER_GENERATION+x}" &&
20193            test -z "${MAGI_WORKER_HOME+x}" &&
20194            test -z "${MAGI_WORKER_EXE+x}" &&
20195            test "$MAGI_HOME/repo" -ef . &&
20196            test "$MAGI_RUN" = verification-parent &&
20197            test "$MAGI_NODE" = gate &&
20198            test "$MAGI_SEAT" = worker-fixture &&
20199            test -n "$MAGI_PROCESS_OWNER" &&
20200            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20201            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20202            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20203            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20204        // Negative control: an ordinary child inherits the contamination.
20205        assert!(
20206            !std::process::Command::new("sh")
20207                .args(["-c", probe])
20208                .current_dir(&repo)
20209                .status()
20210                .unwrap()
20211                .success()
20212        );
20213        let unrelated_home = tempfile::tempdir().unwrap();
20214        assert!(
20215            crate::worker::current_job(unrelated_home.path()).is_err(),
20216            "an inherited worker id cannot be resolved in a unit fixture's home"
20217        );
20218        let mut runner = runner_at(RunStatus::Gating);
20219        runner.state.repo = repo.clone();
20220        runner.state.config.disk.min_free_bytes = 0;
20221        runner.state.config.coordination = crate::config::test_coordination();
20222        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20223        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20224        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20225        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20226        let winner = Candidate {
20227            index: 0,
20228            label: 'A',
20229            agent: "fixture".into(),
20230            branch: "main".into(),
20231            worktree: repo.clone(),
20232            summary: String::new(),
20233            stat: String::new(),
20234            files: 0,
20235            commits: 0,
20236            empty: false,
20237            failed: None,
20238            verified_noop: None,
20239            duration_ms: 0,
20240            folded: false,
20241        };
20242        let (e2e, retried, pids) = run_e2e_with_retry(
20243            &mut runner.state,
20244            &["sh".into(), "-c".into()],
20245            &[format!("{probe} && printf initial")],
20246            &repo,
20247            Duration::from_secs(10),
20248            "initial",
20249        )
20250        .await;
20251        assert!(!retried && pids.is_empty());
20252        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20253        assert_eq!(e2e[0].output_tail, "initial");
20254        let gate = runner.run_gate(&winner).await.unwrap();
20255        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20256        // An interrupted repair must recheck e2e from saved state. Landing
20257        // reuses that state and must run both pre-gate and gate again.
20258        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20259            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20260            passed: false,
20261            outcomes: Vec::new(),
20262        });
20263        runner.state.save().unwrap();
20264        let mut resumed = RunState::load(&runner.state.id).unwrap();
20265        let wait = crate::repo_lock::WaitPolicy::default();
20266        assert!(matches!(
20267            check_gate_repair_e2e(&mut resumed, &winner, &wait)
20268                .await
20269                .unwrap(),
20270            GateFix::Retry
20271        ));
20272        let saved = RunState::load(&resumed.id).unwrap();
20273        let repair = saved.gate_repair_e2e.unwrap();
20274        assert!(repair.passed);
20275        assert_eq!(repair.outcomes.len(), 1);
20276        assert!(repair.outcomes[0].ok());
20277        let landing = run_landing_gate(&mut resumed, &winner, &wait)
20278            .await
20279            .unwrap();
20280        assert_eq!(landing.len(), 1);
20281        assert!(landing[0].ok(), "{landing:?}");
20282        assert_eq!(
20283            std::fs::read_to_string(repo.join("phases")).unwrap(),
20284            "pre\ngate\ne2e\npre\ngate\n"
20285        );
20286        assert_worker();
20287    }
20288
20289    #[cfg(unix)]
20290    #[tokio::test]
20291    async fn verification_keeps_the_panic_and_the_full_output() {
20292        crate::run::pin_test_home();
20293        let tmp = tempfile::tempdir().unwrap();
20294        let mut config = Config::default();
20295        config.disk.min_free_bytes = 0;
20296        let mut state = RunState::new(
20297            tmp.path().to_owned(),
20298            "main".into(),
20299            "base".into(),
20300            "verification evidence".into(),
20301            config,
20302        );
20303        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20304                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
20305                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20306        let mut names = Vec::new();
20307        for _ in 0..2 {
20308            let (outcomes, _) = run_commands(
20309                &mut state,
20310                "gate",
20311                "gate",
20312                0,
20313                &["sh".into(), "-c".into()],
20314                &[script.into()],
20315                tmp.path(),
20316                Duration::from_secs(60),
20317            )
20318            .await;
20319            let tail = &outcomes[0].output_tail;
20320            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20321            assert!(tail.contains("boom assertion"));
20322            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20323            let line = tail.lines().next().unwrap();
20324            names.push(line.to_owned());
20325        }
20326        assert_ne!(names[0], names[1], "the second run must not overwrite");
20327        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20328        assert!(full.len() > 500_000 && full.contains("boom assertion"));
20329        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20330
20331        // Storage failures are evidence too; preserve the command's verdict.
20332        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20333        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20334        let (outcomes, _) = run_commands(
20335            &mut state,
20336            "gate",
20337            "gate",
20338            0,
20339            &["sh".into(), "-c".into()],
20340            &["echo failure-output; exit 7".into()],
20341            tmp.path(),
20342            Duration::from_secs(10),
20343        )
20344        .await;
20345        assert_eq!(outcomes[0].code, Some(7));
20346        assert!(state.events.iter().any(|e| {
20347            e.node == "gate" && e.message.contains("could not save the full gate output:")
20348        }));
20349    }
20350
20351    #[cfg(unix)]
20352    #[tokio::test]
20353    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20354        crate::run::pin_test_home();
20355        let tmp = tempfile::tempdir().unwrap();
20356        let mut config = Config::default();
20357        config.disk.min_free_bytes = 0;
20358        let mut state = RunState::new(
20359            tmp.path().to_owned(),
20360            "main".into(),
20361            "base".into(),
20362            "verification evidence".into(),
20363            config,
20364        );
20365        let (outcomes, pids) = run_commands(
20366            &mut state,
20367            "gate",
20368            "gate",
20369            1,
20370            &["sh".into(), "-c".into()],
20371            &[
20372                "printf before-timeout; exec sleep 30".into(),
20373                "kill -TERM $$".into(),
20374                "exit 7".into(),
20375                "printf success".into(),
20376            ],
20377            tmp.path(),
20378            Duration::from_secs(1),
20379        )
20380        .await;
20381        wait_for_timed_out_children_to_die(&pids).await;
20382        assert_eq!(
20383            pids.len(),
20384            1,
20385            "unleased timeout retains its death-check PID"
20386        );
20387        assert_eq!(outcomes.len(), 4);
20388        assert_eq!(outcomes[0].code, None);
20389        assert!(outcomes[0].output_tail.contains("before-timeout"));
20390        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20391        assert_eq!(outcomes[1].code, None);
20392        assert!(outcomes[1].output_tail.contains("command terminated:"));
20393        assert!(!outcomes[1].output_tail.contains("timed out"));
20394        assert_eq!(outcomes[2].code, Some(7));
20395        assert!(!outcomes[2].output_tail.contains("timed out"));
20396        assert_eq!(outcomes[3].code, Some(0));
20397        assert_eq!(outcomes[3].output_tail, "success");
20398        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20399    }
20400
20401    /// The addendum's second gap: a `verify.gate` command running for real
20402    /// wall-clock time had nothing at all to show for it in `active` before
20403    /// `run_commands` learned to record it — a run could sit in `Gating` for
20404    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
20405    /// what was actually happening. Proven with a genuinely still-running
20406    /// command, not just a before/after check on the final state: a poller
20407    /// task reads the same `run.json` `gate()` is writing, the same way the
20408    /// phone or `magi show` would, while the shell command is still blocked
20409    /// on its own release marker.
20410    #[tokio::test]
20411    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20412        crate::run::pin_test_home();
20413
20414        let tmp = tempfile::tempdir().expect("tempdir");
20415        let repo = tmp.path().join("repo");
20416        std::fs::create_dir_all(&repo).unwrap();
20417        init_repo(&repo);
20418
20419        let mut config = Config::default();
20420        config.disk.min_free_bytes = 0;
20421        config.verify.gate = vec![
20422            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20423             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20424                .to_owned(),
20425        ];
20426
20427        let mut state = RunState::new(
20428            repo.clone(),
20429            "main".to_owned(),
20430            "deadbeef".to_owned(),
20431            "task".to_owned(),
20432            config,
20433        );
20434        let run_id = state.id.clone();
20435        state.candidates = vec![Candidate {
20436            index: 0,
20437            label: 'A',
20438            agent: "alpha".to_owned(),
20439            branch: "does-not-exist".to_owned(),
20440            worktree: repo.clone(),
20441            summary: String::new(),
20442            stat: String::new(),
20443            files: 0,
20444            commits: 0,
20445            empty: false,
20446            failed: None,
20447            verified_noop: None,
20448            duration_ms: 0,
20449            folded: false,
20450        }];
20451        state.tally = Some(Tally {
20452            first_choice: BTreeMap::from([('A', 1)]),
20453            borda: BTreeMap::new(),
20454            winner: 'A',
20455            rankings: 1,
20456            unanimous_initial: true,
20457            deliberated: false,
20458            changed_votes: 0,
20459            unanimous_final: true,
20460            tie_break: None,
20461            judges: 0,
20462            present: 0,
20463            quorum: 0,
20464            met_quorum: true,
20465            uncontested: Some("only candidate A produced a change".to_owned()),
20466        });
20467        state.reviews = vec![ReviewRound {
20468            round: 1,
20469            head: "deadbeef".to_owned(),
20470            verified_head: None,
20471            verified_at: None,
20472            reviews: Vec::new(),
20473            e2e: Vec::new(),
20474            fix: None,
20475            blocking: 0,
20476            answered: 0,
20477            expected: 0,
20478            clean: true,
20479            verify_retried: false,
20480            e2e_deferred: false,
20481            e2e_defer_reason: None,
20482            progressed: false,
20483            vote_split: false,
20484            reconsideration: Vec::new(),
20485            verdict: None,
20486        }];
20487
20488        let mut runner = Runner {
20489            state,
20490            roles: ResolvedRoles {
20491                implementers: Vec::new(),
20492                judges: Vec::new(),
20493                reviewers: Vec::new(),
20494                fixer: None,
20495                conductor: conductor(),
20496                implementer_roster: Vec::new(),
20497                judge_roster: Vec::new(),
20498                reviewer_roster: Vec::new(),
20499            },
20500            sem: Arc::new(Semaphore::new(1)),
20501            pause: Pause::new(),
20502            interrupt: Pause::new(),
20503        };
20504
20505        let started_marker = repo.join("started.marker");
20506        let release_marker = repo.join("release.marker");
20507        let poller = tokio::spawn(async move {
20508            // Bounded so a regression that never records the task entry
20509            // fails this test in seconds instead of hanging the suite —
20510            // the same shape `a_park_requested_while_a_seat_is_mid_call_
20511            // does_not_cut_it_short` uses for the same reason.
20512            for _ in 0..100 {
20513                if started_marker.exists()
20514                    && let Ok(s) = crate::run::RunState::load(&run_id)
20515                    && let Some(a) = s.active.get("gate")
20516                {
20517                    std::fs::write(&release_marker, b"go").expect("release marker");
20518                    return Some(a.clone());
20519                }
20520                tokio::time::sleep(Duration::from_millis(50)).await;
20521            }
20522            None
20523        });
20524
20525        runner.gate().await.expect("gate");
20526        let captured = poller.await.expect("poller task");
20527        let captured = captured.expect(
20528            "the poller never saw a `gate` task entry in run.json while the command was \
20529             still blocked on its own release marker",
20530        );
20531
20532        assert_eq!(captured.task.as_deref(), Some("gate"));
20533        assert_eq!(captured.node, "gate");
20534        assert_eq!(captured.index, Some(1));
20535        assert_eq!(captured.total, Some(1));
20536        assert!(
20537            captured
20538                .command
20539                .as_deref()
20540                .is_some_and(|c| c.contains("started.marker")),
20541            "{captured:?}"
20542        );
20543
20544        assert!(
20545            runner.state.active.is_empty(),
20546            "the entry must be cleared once the command actually finished: {:?}",
20547            runner.state.active
20548        );
20549        assert!(runner.state.gate_ran);
20550        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20551    }
20552
20553    /// The hand-off over a blocking finding a reviewer rejected on leaves a
20554    /// record for `land`; one with only a Minor, or no reject, leaves none.
20555    #[tokio::test]
20556    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20557        use crate::verdict::{Finding, ReviewVote, Severity};
20558        crate::run::pin_test_home();
20559        let tmp = tempfile::tempdir().expect("tempdir");
20560        let repo = tmp.path().join("repo");
20561        std::fs::create_dir_all(&repo).unwrap();
20562        init_repo(&repo);
20563
20564        for (severity, vote, expect) in [
20565            (Severity::Major, ReviewVote::Reject, true),
20566            (Severity::Minor, ReviewVote::Reject, false),
20567            (Severity::Major, ReviewVote::Approve, false),
20568        ] {
20569            let mut round = review_round(false, 1, 1, 1, false, true);
20570            round.reviews = vec![ReviewRecord {
20571                reviewer: 1,
20572                agent: "alpha".to_owned(),
20573                summary: String::new(),
20574                findings: vec![Finding {
20575                    id: "R1-1-1".to_owned(),
20576                    severity,
20577                    file: None,
20578                    line: None,
20579                    title: "t".to_owned(),
20580                    detail: String::new(),
20581                }],
20582                vote: Some(vote),
20583                failed: None,
20584                duration_ms: 0,
20585                attempts: 0,
20586            }];
20587            let mut state = RunState::new(
20588                repo.clone(),
20589                "main".to_owned(),
20590                "deadbeef".to_owned(),
20591                "task".to_owned(),
20592                Config::default(),
20593            );
20594            state.reviews = vec![round];
20595            let mut runner = Runner {
20596                state,
20597                roles: ResolvedRoles {
20598                    implementers: Vec::new(),
20599                    judges: Vec::new(),
20600                    reviewers: Vec::new(),
20601                    fixer: None,
20602                    conductor: conductor(),
20603                    implementer_roster: Vec::new(),
20604                    judge_roster: Vec::new(),
20605                    reviewer_roster: Vec::new(),
20606                },
20607                sem: Arc::new(Semaphore::new(1)),
20608                pause: Pause::new(),
20609                interrupt: Pause::new(),
20610            };
20611            let shell = runner.state.config.shell();
20612            runner
20613                .stop_reviewing("round budget spent", &shell, &repo)
20614                .await
20615                .expect("stop_reviewing");
20616            assert_eq!(runner.state.status, RunStatus::Gating);
20617            assert_eq!(
20618                runner.state.contested_handoff.is_some(),
20619                expect,
20620                "{severity:?} + {vote:?}"
20621            );
20622        }
20623    }
20624
20625    /// The shape the incident this whole fix responds to actually had: the
20626    /// round budget spent, the last round's own e2e blocked on the shared
20627    /// build cache (held here by a live pid — this test process — exactly
20628    /// `cache`'s own unit tests' pattern for "another owner, still alive"
20629    /// without forking a process). `stop_reviewing` must retry it — not
20630    /// silently leave the round looking untouched (the catch-up-only half of
20631    /// the bug), and not read the contention as a red `e2e` and block the
20632    /// run on it (the other half). Called directly, the same way
20633    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
20634    /// above exercises `gate`, so this never needs a real cargo build to
20635    /// reach: the lease is never released, so `with_cache_lease` never gets
20636    /// past acquiring it into anything that would need a real workspace.
20637    #[tokio::test]
20638    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20639        crate::run::pin_test_home();
20640        let home = crate::run::home();
20641
20642        let tmp = tempfile::tempdir().expect("tempdir");
20643        let repo = tmp.path().join("repo");
20644        std::fs::create_dir_all(&repo).unwrap();
20645        init_repo(&repo);
20646        let head = crate::git::rev_parse(&repo, "HEAD")
20647            .await
20648            .expect("rev-parse");
20649        // Unique to this test, so holding its lease cannot collide with
20650        // another test sharing the same process-wide `home`.
20651        let cache_dir = tmp.path().join("target");
20652
20653        let mut config = Config::default();
20654        config.verify.e2e = vec![format!(
20655            "CARGO_TARGET_DIR='{}' test -f README.md",
20656            cache_dir.display()
20657        )];
20658        config.graph.review_rounds = 1;
20659        // Bounded so a regression that does start waiting fails the test in
20660        // seconds, not hangs it.
20661        config.graph.timeout_verify = Some(2);
20662        config.graph.timeout_resource_wait = 2;
20663
20664        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20665        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20666            .expect("no io error acquiring directly")
20667        {
20668            crate::cache::AcquireOutcome::Acquired(g) => g,
20669            crate::cache::AcquireOutcome::Busy(b) => {
20670                panic!("expected the direct acquire to win the lease first: {b:?}")
20671            }
20672        };
20673
20674        let mut state = RunState::new(
20675            repo.clone(),
20676            "main".to_owned(),
20677            head.clone(),
20678            "task".to_owned(),
20679            config,
20680        );
20681        state.candidates = vec![Candidate {
20682            index: 0,
20683            label: 'A',
20684            agent: "alpha".to_owned(),
20685            branch: "does-not-exist".to_owned(),
20686            worktree: repo.clone(),
20687            summary: String::new(),
20688            stat: String::new(),
20689            files: 0,
20690            commits: 0,
20691            empty: false,
20692            failed: None,
20693            verified_noop: None,
20694            duration_ms: 0,
20695            folded: false,
20696        }];
20697        state.tally = Some(Tally {
20698            first_choice: BTreeMap::from([('A', 1)]),
20699            borda: BTreeMap::new(),
20700            winner: 'A',
20701            rankings: 1,
20702            unanimous_initial: true,
20703            deliberated: false,
20704            changed_votes: 0,
20705            unanimous_final: true,
20706            tie_break: None,
20707            judges: 0,
20708            present: 0,
20709            quorum: 0,
20710            met_quorum: true,
20711            uncontested: Some("only candidate A produced a change".to_owned()),
20712        });
20713        // The round budget's last round, deferred: `needs_catchup_run`'s
20714        // other trigger. `stop_reviewing`'s retry machinery must treat this
20715        // exactly like a resource-blocked attempt once it actually runs.
20716        state.reviews = vec![ReviewRound {
20717            round: 1,
20718            head: head.clone(),
20719            verified_head: None,
20720            verified_at: None,
20721            reviews: Vec::new(),
20722            e2e: Vec::new(),
20723            fix: None,
20724            blocking: 1,
20725            answered: 1,
20726            expected: 1,
20727            clean: false,
20728            verify_retried: false,
20729            e2e_deferred: true,
20730            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20731            progressed: false,
20732            vote_split: false,
20733            reconsideration: Vec::new(),
20734            verdict: None,
20735        }];
20736
20737        let mut runner = Runner {
20738            state,
20739            roles: ResolvedRoles {
20740                implementers: Vec::new(),
20741                judges: Vec::new(),
20742                reviewers: Vec::new(),
20743                fixer: None,
20744                conductor: conductor(),
20745                implementer_roster: Vec::new(),
20746                judge_roster: Vec::new(),
20747                reviewer_roster: Vec::new(),
20748            },
20749            sem: Arc::new(Semaphore::new(1)),
20750            pause: Pause::new(),
20751            interrupt: Pause::new(),
20752        };
20753
20754        let shell = runner.state.config.shell();
20755        runner
20756            .stop_reviewing("round budget spent", &shell, &repo)
20757            .await
20758            .expect("stop_reviewing");
20759
20760        let last = runner.state.reviews.last().expect("round record");
20761        assert_eq!(
20762            last.e2e_status(),
20763            E2eStatus::ResourceBlocked,
20764            "the shared cache is still held; the attempt must read as blocked, not deferred or \
20765             failed: {last:?}"
20766        );
20767        assert_eq!(
20768            last.verified_head.as_deref(),
20769            Some(head.as_str()),
20770            "which commit this attempt targeted is known even though nothing finished checking \
20771             it"
20772        );
20773        let first_attempt_at = last
20774            .verified_at
20775            .expect("when this attempt ran is known too");
20776        assert_ne!(
20777            runner.state.status,
20778            RunStatus::Blocked,
20779            "contention is evidence about the machine, not the patch — it must not settle the \
20780             run as blocked: {:?}",
20781            runner.state.status
20782        );
20783        assert!(
20784            !runner
20785                .state
20786                .events
20787                .iter()
20788                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20789            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20790            runner.state.events
20791        );
20792
20793        // The cache is still held: a later reentry must retry the same
20794        // round's verification again — not leave it looking exactly as
20795        // untouched as the first blocked attempt, which is indistinguishable
20796        // from never having tried again at all.
20797        runner
20798            .stop_reviewing("round budget spent", &shell, &repo)
20799            .await
20800            .expect("stop_reviewing retry");
20801        assert_eq!(
20802            runner.state.reviews.len(),
20803            1,
20804            "no new round was started: {:?}",
20805            runner.state.reviews
20806        );
20807        let last = runner.state.reviews.last().expect("round record");
20808        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20809        assert!(
20810            last.verified_at.expect("still known") > first_attempt_at,
20811            "a second reentry must be a fresh attempt, not a stale copy of the first"
20812        );
20813        assert_ne!(runner.state.status, RunStatus::Blocked);
20814
20815        held.release();
20816    }
20817
20818    #[cfg(unix)]
20819    #[tokio::test]
20820    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20821        ask_test_home();
20822        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20823            let tmp = tempfile::tempdir().unwrap();
20824            let repo = tmp.path().join("repo");
20825            std::fs::create_dir(&repo).unwrap();
20826            init_repo(&repo);
20827            let run_git = |args: &[&str]| {
20828                let out = std::process::Command::new("git")
20829                    .args(args)
20830                    .current_dir(&repo)
20831                    .output()
20832                    .unwrap();
20833                assert!(
20834                    out.status.success(),
20835                    "{args:?}: {}",
20836                    String::from_utf8_lossy(&out.stderr)
20837                );
20838            };
20839            let origin = tmp.path().join("origin.git");
20840            run_git(&["init", "--bare", origin.to_str().unwrap()]);
20841            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20842            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20843            run_git(&["add", "spec.rs"]);
20844            run_git(&["commit", "-m", "initial spec"]);
20845            run_git(&["push", "origin", "main"]);
20846            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20847            run_git(&["checkout", "-b", "fix/review"]);
20848            std::fs::write(
20849                repo.join("check.rs"),
20850                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20851            )
20852            .unwrap();
20853            run_git(&["add", "check.rs"]);
20854            run_git(&["commit", "-m", "use spec"]);
20855            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20856            run_git(&["checkout", "main"]);
20857            std::fs::write(
20858                repo.join("spec.rs"),
20859                "struct AgentSpec { max_turn_secs: u64 }\n",
20860            )
20861            .unwrap();
20862            run_git(&["add", "spec.rs"]);
20863            run_git(&["commit", "-m", "extend spec"]);
20864            run_git(&["push", "origin", "main"]);
20865            run_git(&["checkout", "fix/review"]);
20866
20867            let mut runner = runner_at(RunStatus::Reviewing);
20868            runner.state.repo = repo.clone();
20869            runner.state.base_commit = base;
20870            runner.state.pr_head = Some("fix/review".into());
20871            runner.state.config.graph.review_rounds = max_rounds;
20872            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20873            runner.state.config.graph.sessions = false;
20874            runner.state.config.disk.min_free_bytes = 0;
20875            runner.state.config.verify.e2e =
20876                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20877            runner.state.candidates = vec![Candidate {
20878                index: 0,
20879                label: 'A',
20880                agent: "fixture".into(),
20881                branch: "fix/review".into(),
20882                worktree: repo.clone(),
20883                summary: String::new(),
20884                stat: String::new(),
20885                files: 1,
20886                commits: 1,
20887                empty: false,
20888                failed: None,
20889                verified_noop: None,
20890                duration_ms: 0,
20891                folded: false,
20892            }];
20893            runner.state.tally = Some(Tally {
20894                first_choice: BTreeMap::new(),
20895                borda: BTreeMap::new(),
20896                winner: 'A',
20897                rankings: 0,
20898                unanimous_initial: true,
20899                deliberated: false,
20900                changed_votes: 0,
20901                unanimous_final: true,
20902                tie_break: None,
20903                judges: 0,
20904                present: 0,
20905                quorum: 0,
20906                met_quorum: true,
20907                uncontested: Some("review-only".into()),
20908            });
20909            let mut last = review_round(false, 0, 1, 1, false, false);
20910            last.head = reviewed.clone();
20911            last.e2e_deferred = true;
20912            last.e2e[0].resource_blocked = true;
20913            last.e2e[0].output_tail = "waiting for the shared build cache".into();
20914            if max_rounds == 1 {
20915                last.e2e.clear();
20916            }
20917            runner.state.reviews = vec![last];
20918            let calls = tmp.path().join("fix-calls");
20919            let mut fixer = spec("repair");
20920            fixer
20921                .env
20922                .insert("FIX_CALLS".into(), calls.display().to_string());
20923            fixer.command = vec!["sh".into(), "-c".into(), r#"
20924                echo fix >> "$FIX_CALLS"
20925                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20926                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20927                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20928                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20929            "#.into()];
20930            let mut reviewer = spec("reviewer");
20931            reviewer.command = vec![
20932                "sh".into(),
20933                "-c".into(),
20934                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20935                    .into(),
20936            ];
20937            runner.roles.fixer = Some(vec![fixer]);
20938            runner.roles.reviewers = vec![reviewer];
20939
20940            runner.sync_to_base().await.unwrap();
20941            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20942            assert_ne!(rebased, reviewed);
20943            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20944            if reentry {
20945                // Simulate a crash after catch-up evidence was saved but before
20946                // dispatching the fixer. No dedicated pending-repair field exists.
20947                let shell = runner.state.config.shell();
20948                runner
20949                    .stop_unresolved_review("deferred verification", &shell, &repo)
20950                    .await
20951                    .unwrap();
20952                assert!(runner.catchup_repairable());
20953                assert!(!calls.exists());
20954                runner.state = RunState::load(&runner.state.id).unwrap();
20955                assert!(runner.catchup_repairable());
20956            }
20957            runner.review_loop().await.unwrap();
20958            let first = &runner.state.reviews[0];
20959            assert_eq!(first.head, reviewed);
20960            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20961            assert!(
20962                first.verify_retried,
20963                "a reproduced build failure gets exactly one retry: {:?}",
20964                first.e2e
20965            );
20966            assert!(first.e2e[0].output_tail.contains("E0063"));
20967            assert!(!runner.catchup_repairable());
20968            if max_rounds == 2 {
20969                assert_eq!(
20970                    runner.state.status,
20971                    RunStatus::Gating,
20972                    "{}",
20973                    crate::report::run(&runner.state)
20974                );
20975                assert_eq!(runner.state.reviews.len(), 2);
20976                assert!(first.fix.as_ref().unwrap().committed);
20977                assert!(runner.state.reviews[1].clean);
20978                assert_eq!(
20979                    runner
20980                        .state
20981                        .reviews
20982                        .iter()
20983                        .filter(|r| r.fix.is_some())
20984                        .count(),
20985                    1
20986                );
20987                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20988                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20989            } else {
20990                assert_eq!(runner.state.status, RunStatus::Blocked);
20991                assert!(first.fix.is_none());
20992                assert!(!calls.exists());
20993                let recovery = crate::recovery::assess(
20994                    "prior",
20995                    Some(&runner.state),
20996                    &runner.landing_base(),
20997                    crate::recovery::Relation::Diverged,
20998                );
20999                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
21000                let reasons = recovery.reasons.join("\n");
21001                assert!(
21002                    reasons.contains(&reviewed) && reasons.contains(&rebased),
21003                    "{reasons}"
21004                );
21005                assert!(
21006                    reasons.contains("base sync to")
21007                        && reasons.contains("additional review is required"),
21008                    "{reasons}"
21009                );
21010            }
21011        }
21012    }
21013
21014    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
21015    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
21016    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
21017    /// own top-of-function fast path (`review_conclusion`) correctly reads
21018    /// this shape as `None` rather than guessing `Blocked`, and the loop's
21019    /// own `for` range is empty once the round budget is spent, so
21020    /// `review_loop` must retry the check itself rather than silently doing
21021    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
21022    /// above exercises directly, but through `review_loop`'s own entry point
21023    /// this time, proving the wiring between the two rather than just the
21024    /// retry logic in isolation.
21025    #[tokio::test]
21026    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21027        crate::run::pin_test_home();
21028        let home = crate::run::home();
21029
21030        let tmp = tempfile::tempdir().expect("tempdir");
21031        let repo = tmp.path().join("repo");
21032        std::fs::create_dir_all(&repo).unwrap();
21033        init_repo(&repo);
21034        let head = crate::git::rev_parse(&repo, "HEAD")
21035            .await
21036            .expect("rev-parse");
21037        let cache_dir = tmp.path().join("target");
21038
21039        let mut config = Config::default();
21040        config.verify.e2e = vec![format!(
21041            "CARGO_TARGET_DIR='{}' test -f README.md",
21042            cache_dir.display()
21043        )];
21044        config.graph.review_rounds = 1;
21045        config.graph.timeout_verify = Some(2);
21046        config.graph.timeout_resource_wait = 2;
21047
21048        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21049        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21050            .expect("no io error acquiring directly")
21051        {
21052            crate::cache::AcquireOutcome::Acquired(g) => g,
21053            crate::cache::AcquireOutcome::Busy(b) => {
21054                panic!("expected the direct acquire to win the lease first: {b:?}")
21055            }
21056        };
21057
21058        let mut state = RunState::new(
21059            repo.clone(),
21060            "main".to_owned(),
21061            head.clone(),
21062            "task".to_owned(),
21063            config,
21064        );
21065        state.candidates = vec![Candidate {
21066            index: 0,
21067            label: 'A',
21068            agent: "alpha".to_owned(),
21069            branch: "does-not-exist".to_owned(),
21070            worktree: repo.clone(),
21071            summary: String::new(),
21072            stat: String::new(),
21073            files: 0,
21074            commits: 0,
21075            empty: false,
21076            failed: None,
21077            verified_noop: None,
21078            duration_ms: 0,
21079            folded: false,
21080        }];
21081        state.tally = Some(Tally {
21082            first_choice: BTreeMap::from([('A', 1)]),
21083            borda: BTreeMap::new(),
21084            winner: 'A',
21085            rankings: 1,
21086            unanimous_initial: true,
21087            deliberated: false,
21088            changed_votes: 0,
21089            unanimous_final: true,
21090            tie_break: None,
21091            judges: 0,
21092            present: 0,
21093            quorum: 0,
21094            met_quorum: true,
21095            uncontested: Some("only candidate A produced a change".to_owned()),
21096        });
21097        // The exact shape a prior process's `stop_reviewing` would have left
21098        // on disk: the round budget's last round, a real attempt already
21099        // made and already resource-blocked.
21100        state.reviews = vec![ReviewRound {
21101            round: 1,
21102            head: head.clone(),
21103            verified_head: Some(head.clone()),
21104            verified_at: Some(jiff::Timestamp::now()),
21105            reviews: Vec::new(),
21106            e2e: vec![CommandOutcome {
21107                command: format!(
21108                    "CARGO_TARGET_DIR='{}' test -f README.md",
21109                    cache_dir.display()
21110                ),
21111                code: None,
21112                output_tail: "waiting for the shared build cache".to_owned(),
21113                duration_ms: 0,
21114                resource_blocked: true,
21115            }],
21116            fix: None,
21117            blocking: 1,
21118            answered: 1,
21119            expected: 1,
21120            clean: false,
21121            verify_retried: false,
21122            e2e_deferred: false,
21123            e2e_defer_reason: None,
21124            progressed: false,
21125            vote_split: false,
21126            reconsideration: Vec::new(),
21127            verdict: None,
21128        }];
21129
21130        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
21131        let mut runner = Runner {
21132            state,
21133            roles: ResolvedRoles {
21134                implementers: Vec::new(),
21135                judges: Vec::new(),
21136                reviewers: Vec::new(),
21137                fixer: None,
21138                conductor: conductor(),
21139                implementer_roster: Vec::new(),
21140                judge_roster: Vec::new(),
21141                reviewer_roster: Vec::new(),
21142            },
21143            sem: Arc::new(Semaphore::new(1)),
21144            pause: Pause::new(),
21145            interrupt: Pause::new(),
21146        };
21147
21148        // The lease is still held throughout, so this reentry's own retry is
21149        // also contended — proving `review_loop` actually tried again (not
21150        // that it happened to succeed) is what the timestamp comparison
21151        // below is for.
21152        runner.review_loop().await.expect("review_loop");
21153
21154        assert_eq!(
21155            runner.state.reviews.len(),
21156            1,
21157            "no new round was started on top of the unresolved one: {:?}",
21158            runner.state.reviews
21159        );
21160        let last = &runner.state.reviews[0];
21161        assert_eq!(
21162            last.e2e_status(),
21163            E2eStatus::ResourceBlocked,
21164            "still contended: {last:?}"
21165        );
21166        assert!(
21167            last.verified_at.expect("still known") > first_attempt_at,
21168            "review_loop must have actually retried the check, not left it exactly as found"
21169        );
21170        assert_ne!(
21171            runner.state.status,
21172            RunStatus::Blocked,
21173            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21174            runner.state.status
21175        );
21176
21177        held.release();
21178    }
21179
21180    #[tokio::test]
21181    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21182        crate::run::pin_test_home();
21183        for status in [RunStatus::Blocked, RunStatus::Gating] {
21184            let mut runner = runner_at(status);
21185            runner.state.config.graph.land = true;
21186            runner.state.gate_ran = true;
21187            runner.state.merge = Some(MergeOutcome {
21188                mode: MergeMode::Pr,
21189                ok: false,
21190                detail: "the fixer produced no commit".into(),
21191                empty: false,
21192            });
21193            let url = "https://example.invalid/o/r/pull/1";
21194            runner.state.pr = Some(crate::run::PrRecord {
21195                url: url.into(),
21196                number: 1,
21197                state: "open".into(),
21198                checks: "red".into(),
21199                round: 1,
21200                rounds: 1,
21201                red_at_merge: Vec::new(),
21202            });
21203            runner.state.save().unwrap();
21204            // The nonexistent repository makes observation fail before a forge
21205            // command can run. We still must reach land, rather than silently
21206            // return from the graph with its old gate and merge records.
21207            runner.execute_graph().await.unwrap();
21208            assert_eq!(runner.state.status, RunStatus::Blocked);
21209            assert!(runner.state.land_sync_pending);
21210            assert!(runner.state.events.iter().any(|event|
21211                event.node == "land" && event.message == format!("watching {url}")));
21212            assert_eq!(
21213                runner.state.merge.as_ref().unwrap().detail,
21214                "the fixer produced no commit"
21215            );
21216        }
21217    }
21218
21219    #[tokio::test]
21220    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21221        crate::run::pin_test_home();
21222        let tmp = tempfile::tempdir().expect("tempdir");
21223        let repo = tmp.path().join("repo");
21224        std::fs::create_dir_all(&repo).unwrap();
21225        init_repo(&repo);
21226
21227        let mut config = Config::default();
21228        config.merge.mode = MergeMode::Pr;
21229        config.graph.land = true;
21230        config.graph.land_approval = false;
21231
21232        let mut state = RunState::new(
21233            repo.clone(),
21234            "main".to_owned(),
21235            "deadbeef".to_owned(),
21236            "task".to_owned(),
21237            config,
21238        );
21239        state.candidates = vec![Candidate {
21240            index: 0,
21241            label: 'A',
21242            agent: "alpha".to_owned(),
21243            branch: "does-not-exist".to_owned(),
21244            worktree: repo.clone(),
21245            summary: String::new(),
21246            stat: String::new(),
21247            files: 0,
21248            commits: 0,
21249            empty: false,
21250            failed: None,
21251            verified_noop: None,
21252            duration_ms: 0,
21253            folded: false,
21254        }];
21255        state.tally = Some(Tally {
21256            first_choice: BTreeMap::from([('A', 1)]),
21257            borda: BTreeMap::new(),
21258            winner: 'A',
21259            rankings: 1,
21260            unanimous_initial: true,
21261            deliberated: false,
21262            changed_votes: 0,
21263            unanimous_final: true,
21264            tie_break: None,
21265            judges: 0,
21266            present: 0,
21267            quorum: 0,
21268            met_quorum: true,
21269            uncontested: Some("only candidate A produced a change".to_owned()),
21270        });
21271        state.reviews = vec![ReviewRound {
21272            round: 1,
21273            head: "deadbeef".to_owned(),
21274            verified_head: None,
21275            verified_at: None,
21276            reviews: Vec::new(),
21277            e2e: Vec::new(),
21278            fix: None,
21279            blocking: 0,
21280            answered: 0,
21281            expected: 0,
21282            clean: true,
21283            verify_retried: false,
21284            e2e_deferred: false,
21285            e2e_defer_reason: None,
21286            progressed: false,
21287            vote_split: false,
21288            reconsideration: Vec::new(),
21289            verdict: None,
21290        }];
21291        state.gate = vec![CommandOutcome {
21292            command: "test".to_owned(),
21293            code: Some(0),
21294            output_tail: String::new(),
21295            duration_ms: 0,
21296            resource_blocked: false,
21297        }];
21298        state.gate_ran = true;
21299        // A first pass through `merge` already pushed and opened this pull
21300        // request; `status` is `Landing` because a previous call into `land`
21301        // parked or was interrupted before it reached a terminal outcome.
21302        state.status = RunStatus::Landing;
21303        state.merge = Some(MergeOutcome {
21304            mode: MergeMode::Pr,
21305            ok: true,
21306            detail: "https://example.invalid/x/y/pull/1".to_owned(),
21307            empty: false,
21308        });
21309
21310        // The Landing-resume shortcut calls `run_land` directly rather than
21311        // through `merge`, which is exactly the call site that used to skip
21312        // `settle_questions` - see the fixture below.
21313        ask_test_home();
21314        // A real resume loads a persisted run; driver ownership is checked
21315        // against that record before the Landing shortcut is entered.
21316        state.save().expect("save landing run");
21317        let store = ask::Questions::open();
21318        let q = ask_open_question(&store, &state.id);
21319
21320        let mut runner = Runner {
21321            state,
21322            roles: ResolvedRoles {
21323                implementers: Vec::new(),
21324                judges: Vec::new(),
21325                reviewers: Vec::new(),
21326                fixer: None,
21327                conductor: conductor(),
21328                implementer_roster: Vec::new(),
21329                judge_roster: Vec::new(),
21330                reviewer_roster: Vec::new(),
21331            },
21332            sem: Arc::new(Semaphore::new(1)),
21333            pause: Pause::new(),
21334            interrupt: Pause::new(),
21335        };
21336
21337        // `execute`, not `merge` directly: the Landing-resume shortcut lives
21338        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
21339        // exactly because `review_loop` would otherwise clobber the marker
21340        // first.
21341        runner.execute().await.expect("execute");
21342
21343        assert_eq!(
21344            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21345            Some("https://example.invalid/x/y/pull/1"),
21346            "reentry must not push again or open a second pull request over the \
21347             one `land` is already watching"
21348        );
21349        assert_ne!(
21350            runner.state.status,
21351            RunStatus::Landing,
21352            "land could not actually reach the fake pull request, so it must \
21353             have given up rather than left the run silently parked forever"
21354        );
21355        // `land` could not reach the fake pull request, so it gave up into
21356        // `Blocked` - still resumable, so the question must not have been
21357        // swept just because this branch now also calls `settle_questions`.
21358        assert_eq!(runner.state.status, RunStatus::Blocked);
21359        assert!(
21360            store.get(&q.id).unwrap().status.open(),
21361            "Blocked is still alive; settle_questions must have been a no-op here"
21362        );
21363    }
21364
21365    fn state_with_round(round: ReviewRound) -> RunState {
21366        let mut s = RunState::new(
21367            PathBuf::from("/repo"),
21368            "main".to_owned(),
21369            "abc1234".to_owned(),
21370            "add retries".to_owned(),
21371            Config::default(),
21372        );
21373        s.reviews = vec![round];
21374        s
21375    }
21376
21377    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21378        crate::verdict::Finding {
21379            id: id.to_owned(),
21380            severity,
21381            file: None,
21382            line: None,
21383            title: title.to_owned(),
21384            detail: String::new(),
21385        }
21386    }
21387
21388    #[test]
21389    fn pr_body_names_open_findings_and_declined_ones() {
21390        let round = ReviewRound {
21391            round: 2,
21392            head: "deadbee".to_owned(),
21393            verified_head: None,
21394            verified_at: None,
21395            reviews: vec![ReviewRecord {
21396                attempts: 0,
21397                reviewer: 1,
21398                agent: "alpha".to_owned(),
21399                summary: String::new(),
21400                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21401                vote: None,
21402                failed: None,
21403                duration_ms: 0,
21404            }],
21405            e2e: vec![CommandOutcome {
21406                command: "cargo test".to_owned(),
21407                code: Some(0),
21408                output_tail: String::new(),
21409                duration_ms: 0,
21410                resource_blocked: false,
21411            }],
21412            verify_retried: false,
21413            e2e_deferred: false,
21414            e2e_defer_reason: None,
21415            fix: Some(FixRecord {
21416                agent: "alpha".to_owned(),
21417                addressed: Vec::new(),
21418                rejected: vec![crate::verdict::Rejection {
21419                    id: "R1-1-1".to_owned(),
21420                    why: "not reachable from any caller".to_owned(),
21421                }],
21422                notes: String::new(),
21423                committed: true,
21424                failed: None,
21425                duration_ms: 0,
21426                continuation: None,
21427            }),
21428            blocking: 0,
21429            answered: 1,
21430            expected: 1,
21431            clean: false,
21432            progressed: true,
21433            vote_split: false,
21434            reconsideration: Vec::new(),
21435            verdict: None,
21436        };
21437        let state = state_with_round(round);
21438        let body = pr_message(&state, 'A').body;
21439
21440        assert!(body.contains("add retries"), "the task must still be there");
21441        assert!(body.contains("R2-1-1"), "{body}");
21442        assert!(!body.contains("unused import"), "{body}");
21443        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21444        assert!(
21445            !body.contains("not reachable from any caller"),
21446            "the reason it was declined: {body}"
21447        );
21448    }
21449
21450    #[test]
21451    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21452        let round = ReviewRound {
21453            round: 1,
21454            head: "deadbee".to_owned(),
21455            verified_head: None,
21456            verified_at: None,
21457            reviews: vec![ReviewRecord {
21458                attempts: 0,
21459                reviewer: 1,
21460                agent: "alpha".to_owned(),
21461                summary: String::new(),
21462                findings: Vec::new(),
21463                vote: None,
21464                failed: None,
21465                duration_ms: 0,
21466            }],
21467            e2e: Vec::new(),
21468            verify_retried: false,
21469            e2e_deferred: false,
21470            e2e_defer_reason: None,
21471            fix: None,
21472            blocking: 0,
21473            answered: 1,
21474            expected: 1,
21475            clean: true,
21476            progressed: false,
21477            vote_split: false,
21478            reconsideration: Vec::new(),
21479            verdict: None,
21480        };
21481        let state = state_with_round(round);
21482        let body = pr_message(&state, 'A').body;
21483        assert!(!body.contains("Open review findings"), "{body}");
21484        assert!(!body.contains("Declined"), "{body}");
21485    }
21486
21487    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21488        let mut state = RunState::new(
21489            PathBuf::from("/repo"),
21490            "main".to_owned(),
21491            "abc1234".to_owned(),
21492            instruction.to_owned(),
21493            Config::default(),
21494        );
21495        // These fixtures test execution flow, independent of host disk capacity.
21496        state.config.disk.min_free_bytes = 0;
21497        state.candidates.push(Candidate {
21498            index: 0,
21499            label: 'A',
21500            agent: "alpha".to_owned(),
21501            branch: "magi/x/A".to_owned(),
21502            worktree: PathBuf::from("/wt"),
21503            summary: summary.to_owned(),
21504            stat: String::new(),
21505            files: 1,
21506            commits: 1,
21507            empty: false,
21508            failed: None,
21509            verified_noop: None,
21510            folded: false,
21511            duration_ms: 0,
21512        });
21513        state
21514    }
21515
21516    fn review_state(subjects: &[&str]) -> RunState {
21517        let mut state = state_with_summary(
21518            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21519            "",
21520        );
21521        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21522        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21523        state
21524    }
21525
21526    /// A branch rebased onto a main that moved past the recorded
21527    /// `base_commit` is titled from its own first commit, never main's.
21528    #[tokio::test]
21529    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21530        ask_test_home();
21531        let tmp = tempfile::tempdir().unwrap();
21532        let repo = tmp.path().join("repo");
21533        std::fs::create_dir_all(&repo).unwrap();
21534        init_repo(&repo);
21535        let origin = tmp.path().join("origin.git");
21536        let g = |dir: &Path, args: &[&str]| {
21537            let out = std::process::Command::new("git")
21538                .args(args)
21539                .current_dir(dir)
21540                .quiet()
21541                .output()
21542                .expect("spawn git");
21543            assert!(
21544                out.status.success(),
21545                "git {args:?}: {}",
21546                String::from_utf8_lossy(&out.stderr)
21547            );
21548        };
21549        g(
21550            tmp.path(),
21551            &[
21552                "clone",
21553                "--bare",
21554                "-q",
21555                repo.to_str().unwrap(),
21556                origin.to_str().unwrap(),
21557            ],
21558        );
21559        g(
21560            &repo,
21561            &["remote", "add", "origin", origin.to_str().unwrap()],
21562        );
21563        let c1 = git::rev_parse(&repo, "main").await.unwrap();
21564
21565        // Main moves on; the branch is built on top of the new main.
21566        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21567        g(&repo, &["add", "-A"]);
21568        g(
21569            &repo,
21570            &["commit", "-q", "-m", "chore(deps): update a crate"],
21571        );
21572        g(&repo, &["push", "-q", "origin", "main"]);
21573        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21574        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21575        g(&repo, &["add", "-A"]);
21576        g(
21577            &repo,
21578            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21579        );
21580        g(&repo, &["checkout", "-q", "main"]);
21581
21582        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21583        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21584        // Without a readable tracking ref the recorded base's merge base is used.
21585        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21586        assert_eq!(fallback, c1);
21587
21588        let mut state = review_state(&[
21589            "chore(deps): update a crate",
21590            "fix(daemon): apply a chosen action",
21591        ]);
21592        state.repo = repo.clone();
21593        state.base_branch = "main".to_owned();
21594        state.base_commit = c1;
21595        refresh_reviewed_commits(&mut state, "feat/own").await;
21596        assert_eq!(
21597            state.reviewed_commits,
21598            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21599        );
21600        assert_eq!(
21601            review_title(&state).as_deref(),
21602            Some("fix(daemon): apply a chosen action")
21603        );
21604        assert_eq!(
21605            leaked_subjects(&state, "feat/own").await,
21606            Some(vec!["chore(deps): update a crate".to_owned()])
21607        );
21608        // A stale tracking ref is still used when the fetch fails, but the
21609        // leak list is withheld.
21610        state.config.merge.remote = "nowhere".to_owned();
21611        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21612    }
21613
21614    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
21615    /// with `origin/main` one commit behind it, and a summarizing agent that
21616    /// prints whatever `reply` holds.
21617    struct SummaryFixture {
21618        _tmp: tempfile::TempDir,
21619        repo: PathBuf,
21620        reply: PathBuf,
21621        runner: Runner,
21622    }
21623
21624    fn summary_fixture() -> SummaryFixture {
21625        ask_test_home();
21626        let tmp = tempfile::tempdir().unwrap();
21627        let repo = tmp.path().join("repo");
21628        std::fs::create_dir_all(&repo).unwrap();
21629        init_repo(&repo);
21630        let origin = tmp.path().join("origin.git");
21631        let g = |args: &[&str]| {
21632            let out = std::process::Command::new("git")
21633                .args(args)
21634                .current_dir(&repo)
21635                .quiet()
21636                .output()
21637                .expect("spawn git");
21638            assert!(
21639                out.status.success(),
21640                "git {args:?}: {}",
21641                String::from_utf8_lossy(&out.stderr)
21642            );
21643        };
21644        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21645        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21646        g(&["push", "-q", "origin", "main"]);
21647        g(&["checkout", "-q", "-b", "feat/own"]);
21648        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21649        g(&["add", "-A"]);
21650        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21651        let reply = tmp.path().join("reply.txt");
21652        std::fs::write(
21653            &reply,
21654            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21655        )
21656        .unwrap();
21657        let mut runner = runner_at(RunStatus::Gating);
21658        let mut state = review_state(&["feat(own): add the own file"]);
21659        state.repo = repo.clone();
21660        state.base_branch = "main".to_owned();
21661        state.candidates[0].branch = "feat/own".to_owned();
21662        state.candidates[0].worktree = repo.clone();
21663        let mut author = spec("alpha");
21664        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21665        author
21666            .env
21667            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21668        state.config.agents = vec![author];
21669        runner.state = state;
21670        SummaryFixture {
21671            _tmp: tmp,
21672            repo,
21673            reply,
21674            runner,
21675        }
21676    }
21677
21678    async fn posted(f: &mut SummaryFixture) -> PrMessage {
21679        let winner = f.runner.state.candidates[0].clone();
21680        f.runner
21681            .guarded_pr_message(&winner, None, true)
21682            .await
21683            .unwrap()
21684            .expect("an English description never parks")
21685    }
21686
21687    fn clean_round_with_minor_findings() -> ReviewRound {
21688        let mut round = passing_round(2, None);
21689        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21690        round.reviews = vec![ReviewRecord {
21691            attempts: 0,
21692            reviewer: 1,
21693            agent: "alpha".into(),
21694            summary: "内部レビュー".into(),
21695            findings: vec![
21696                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21697                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21698            ],
21699            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21700            failed: None,
21701            duration_ms: 0,
21702        }];
21703        round
21704    }
21705
21706    #[tokio::test]
21707    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21708        for review_only in [false, true] {
21709            let mut f = summary_fixture();
21710            if !review_only {
21711                f.runner.state.candidates[0].agent = "alpha".into();
21712            }
21713            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21714            assert!(f.runner.state.reviews[0].clean);
21715            assert!(f.runner.state.reviews[0].fix.is_none());
21716            assert!(f.runner.state.open_findings().is_empty());
21717            assert_eq!(
21718                crate::pr_outcome::required_finding_ids(&f.runner.state),
21719                ["R2-1-1", "R2-1-2"]
21720            );
21721            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21722            let evidence = crate::pr_outcome::summary_evidence(
21723                &f.runner.state,
21724                crate::pr_outcome::FinalHead::At(&head),
21725            );
21726            assert!(
21727                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21728            );
21729            // The ordinary reply omits both IDs: no cached description is adopted.
21730            let omitted = posted(&mut f).await;
21731            assert!(omitted.body.contains("No description"));
21732            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21733            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21734            assert!(f.runner.state.change_summary.is_none());
21735            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21736            let described = posted(&mut f).await;
21737            assert!(described.body.contains("improve documentation for callers"));
21738            assert!(described.body.contains("remaining edge case"));
21739            assert!(described.body.contains("## Open review findings"));
21740            assert!(!described.body.contains("未修正"));
21741        }
21742    }
21743
21744    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21745        let mut f = summary_fixture();
21746        if !review_only {
21747            f.runner.state.candidates[0].agent = "alpha".into();
21748        }
21749        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21750        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21751        fixed
21752            .fix
21753            .as_mut()
21754            .unwrap()
21755            .rejected
21756            .push(crate::verdict::Rejection {
21757                id: "R1-1-1".into(),
21758                why: "呼び出されないコードのため辞退".into(),
21759            });
21760        let mut clean = clean_round_with_minor_findings();
21761        clean.verified_head = Some(head.clone());
21762        f.runner.state.reviews = vec![fixed, clean];
21763        ok_gate(&mut f.runner.state, Some(&head));
21764        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21765        f.runner.state.config.agents.push(scripted("beta", reply));
21766        f.runner.state.config.roles.language_judge =
21767            Some(crate::config::AgentChoice::One("beta".into()));
21768        f
21769    }
21770
21771    #[tokio::test]
21772    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21773        for review_only in [false, true] {
21774            let mut f = rewrite_fixture_with_findings(review_only,
21775                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21776            let m = posted(&mut f).await;
21777            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21778            let expected =
21779                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21780            assert!(m.body.contains(&expected));
21781            assert!(m.body.contains("## Review rounds"));
21782            assert!(
21783                m.body
21784                    .contains("Gate (`verify.gate`): passed on the final commit")
21785            );
21786            assert!(
21787                m.body
21788                    .contains("Tests (`verify.e2e`): passed on the final commit")
21789            );
21790            let c = f.runner.state.change_summary.as_ref().unwrap();
21791            assert!(
21792                m.body
21793                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21794            );
21795            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21796            assert!(crate::pr_outcome::covers_required_findings(
21797                &f.runner.state,
21798                &m.body
21799            ));
21800            assert!(m.body.contains("documentation remains incomplete"));
21801            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21802            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21803            assert!(f.runner.state.github_text.is_none());
21804        }
21805    }
21806
21807    #[tokio::test]
21808    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21809        for review_only in [false, true] {
21810            for reply in [
21811                "TITLE: fix: public description\nA short English description.",
21812                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21813            ] {
21814                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21815                let winner = f.runner.state.candidates[0].clone();
21816                let posted = f
21817                    .runner
21818                    .guarded_pr_message(&winner, None, true)
21819                    .await
21820                    .unwrap();
21821                assert!(posted.is_none());
21822                assert!(f.runner.state.parked);
21823                let withheld = f.runner.state.github_text.as_ref().unwrap();
21824                assert!(
21825                    withheld.rewrite.is_none(),
21826                    "an incomplete rewrite must not be selectable"
21827                );
21828                let q = ask::Questions::open()
21829                    .get(withheld.question.as_ref().unwrap())
21830                    .unwrap();
21831                assert!(
21832                    !q.choices
21833                        .contains(&crate::github_text::USE_REWRITE.to_owned())
21834                );
21835                assert!(
21836                    q.detail.contains("R1-1-1")
21837                        && q.detail.contains("R2-1-1")
21838                        && q.detail.contains("R2-1-2")
21839                );
21840            }
21841        }
21842    }
21843
21844    #[tokio::test]
21845    async fn japanese_reports_stay_local_in_both_public_run_modes() {
21846        for review_only in [false, true] {
21847            let mut f = summary_fixture();
21848            if !review_only {
21849                f.runner.state.candidates[0].agent = "alpha".into();
21850            }
21851            f.runner.state.instruction = "内部タスクの説明".into();
21852            f.runner.state.candidates[0].summary = "内部の候補報告".into();
21853            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21854            f.runner.state.reviews = vec![
21855                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21856                round_with_notes(2, Some("二回目の内部修正報告です。")),
21857                passing_round(3, Some(&head)),
21858            ];
21859            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21860            ok_gate(&mut f.runner.state, Some(&head));
21861            let captured = f.repo.join("public-prompt.txt");
21862            let author = &mut f.runner.state.config.agents[0];
21863            author.command = vec![
21864                "sh".into(),
21865                "-c".into(),
21866                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21867            ];
21868            author.env.insert(
21869                "PROMPT_FILE".into(),
21870                captured.to_string_lossy().into_owned(),
21871            );
21872            let m = posted(&mut f).await;
21873            assert!(m.body.contains("Adds `own.txt`"));
21874            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21875            assert!(m.body.contains("R1-1-1"));
21876            assert!(
21877                m.body
21878                    .contains("Gate (`verify.gate`): passed on the final commit")
21879            );
21880            assert!(
21881                m.body
21882                    .contains("Tests (`verify.e2e`): passed on the final commit")
21883            );
21884            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21885            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21886            let prompt = std::fs::read_to_string(captured).unwrap();
21887            assert!(prompt.contains(&format!(
21888                "git diff {} {head}",
21889                f.runner.state.change_summary.as_ref().unwrap().base
21890            )));
21891            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21892            assert_eq!(
21893                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21894                "二回目の内部修正報告です。"
21895            );
21896        }
21897    }
21898
21899    #[tokio::test]
21900    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21901        for review_only in [false, true] {
21902            let mut f = summary_fixture();
21903            if !review_only {
21904                f.runner.state.candidates[0].agent = "alpha".into();
21905            }
21906            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21907            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21908            std::fs::write(&f.reply, "no usable description").unwrap();
21909            let failed = posted(&mut f).await;
21910            assert!(
21911                failed
21912                    .body
21913                    .contains("No description of what this change does could be generated")
21914            );
21915            assert!(failed.body.is_ascii());
21916            assert!(f.runner.state.github_text.is_none());
21917            assert!(f.runner.state.change_summary.is_none());
21918            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21919            let evidence = crate::pr_outcome::summary_evidence(
21920                &f.runner.state,
21921                crate::pr_outcome::FinalHead::At(&head),
21922            );
21923            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21924            assert!(
21925                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21926                    .exists()
21927            );
21928            f.runner.state =
21929                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21930            std::fs::write(
21931                &f.reply,
21932                "TITLE: feat(own): add the own file\nAdds the committed own file.",
21933            )
21934            .unwrap();
21935            let retried = posted(&mut f).await;
21936            assert!(retried.body.contains("Adds the committed own file."));
21937            assert_eq!(
21938                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21939                "日本語の修正報告"
21940            );
21941        }
21942    }
21943
21944    #[tokio::test]
21945    async fn same_head_verification_and_findings_invalidate_public_prose() {
21946        let mut f = summary_fixture();
21947        posted(&mut f).await;
21948        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21949        let original = f
21950            .runner
21951            .state
21952            .change_summary
21953            .as_ref()
21954            .unwrap()
21955            .evidence
21956            .clone();
21957        ok_gate(&mut f.runner.state, Some(&head));
21958        std::fs::write(
21959            &f.reply,
21960            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21961        )
21962        .unwrap();
21963        let verified = posted(&mut f).await;
21964        assert!(verified.body.contains("now verified"));
21965        assert_ne!(
21966            f.runner.state.change_summary.as_ref().unwrap().evidence,
21967            original
21968        );
21969        let mut round = round_with_notes(1, Some("内部報告"));
21970        round.reviews.push(ReviewRecord {
21971            attempts: 0,
21972            reviewer: 1,
21973            agent: "alpha".into(),
21974            summary: "内部レビュー".into(),
21975            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21976            vote: None,
21977            failed: None,
21978            duration_ms: 0,
21979        });
21980        round
21981            .fix
21982            .as_mut()
21983            .unwrap()
21984            .rejected
21985            .push(crate::verdict::Rejection {
21986                id: "R1-1-1".into(),
21987                why: "呼び出し元から到達できない".into(),
21988            });
21989        f.runner.state.reviews = vec![round];
21990        // A description omitting the required ID is unusable, even with a title.
21991        let omitted = posted(&mut f).await;
21992        assert!(omitted.body.contains("No description"));
21993        assert!(omitted.body.contains("R1-1-1"));
21994        assert!(!omitted.body.contains("到達"));
21995        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21996        let described = posted(&mut f).await;
21997        assert!(described.body.contains("unreachable from callers"));
21998        assert!(described.body.contains("R1-1-1"));
21999        assert!(!described.body.contains("到達"));
22000        let evidence = f
22001            .runner
22002            .state
22003            .change_summary
22004            .as_ref()
22005            .unwrap()
22006            .evidence
22007            .clone();
22008        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
22009        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
22010        assert!(posted(&mut f).await.body.contains("rationale updated"));
22011        assert_ne!(
22012            f.runner.state.change_summary.as_ref().unwrap().evidence,
22013            evidence
22014        );
22015        // Old serialized summaries have no evidence fingerprint and must regenerate.
22016        f.runner
22017            .state
22018            .change_summary
22019            .as_mut()
22020            .unwrap()
22021            .evidence
22022            .clear();
22023        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22024        assert!(posted(&mut f).await.body.contains("current specification"));
22025    }
22026
22027    #[tokio::test]
22028    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22029        let mut f = summary_fixture();
22030        let m = posted(&mut f).await;
22031        assert_eq!(m.title, "feat(own): add the own file");
22032        assert!(
22033            m.body
22034                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22035            "{}",
22036            m.body
22037        );
22038        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22039        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22040        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22041        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22042        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22043        let kept = f.runner.state.change_summary.as_ref().unwrap();
22044        assert_eq!(
22045            (kept.head.as_str(), kept.base.as_str()),
22046            (head.as_str(), base.as_str())
22047        );
22048        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22049    }
22050
22051    #[tokio::test]
22052    async fn review_only_summary_receives_the_configured_github_language() {
22053        let mut f = summary_fixture();
22054        f.runner.state.config.graph.github_language = "ja".into();
22055        let captured = f.repo.join("summary-prompt.txt");
22056        let author = &mut f.runner.state.config.agents[0];
22057        author.command = vec![
22058            "sh".into(),
22059            "-c".into(),
22060            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22061        ];
22062        author.env.insert(
22063            "PROMPT_FILE".into(),
22064            captured.to_string_lossy().into_owned(),
22065        );
22066        std::fs::write(
22067            &f.reply,
22068            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22069        )
22070        .unwrap();
22071        let m = posted(&mut f).await;
22072        let prompt = std::fs::read_to_string(captured).unwrap();
22073        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22074        assert!(!prompt.contains("in English"));
22075        assert_eq!(m.title, "feat(own): ファイルを追加");
22076        assert!(m.body.contains("own.txt を追加します。"));
22077    }
22078
22079    #[tokio::test]
22080    async fn review_fixes_come_after_the_whole_change_summary() {
22081        let mut f = summary_fixture();
22082        f.runner.state.reviews = vec![
22083            round_with_notes(1, Some("handled the PATH case")),
22084            round_with_notes(2, None),
22085        ];
22086        let m = posted(&mut f).await;
22087        let summary = m.body.find("## Summary").expect("summary");
22088        let fixes = m.body.find("## Review rounds").expect("review rounds");
22089        assert!(summary < fixes, "{}", m.body);
22090        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22091        assert!(m.body.contains("## Final verification"), "{}", m.body);
22092    }
22093
22094    #[tokio::test]
22095    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22096        let mut f = summary_fixture();
22097        let first = posted(&mut f).await;
22098        let first_head = f.runner.state.change_summary.clone().unwrap().head;
22099        // The same head reuses the record: a different reply is not read.
22100        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22101        let again = posted(&mut f).await;
22102        assert_eq!((again.title, again.body), (first.title, first.body));
22103
22104        let g = |args: &[&str]| {
22105            let out = std::process::Command::new("git")
22106                .args(args)
22107                .current_dir(&f.repo)
22108                .quiet()
22109                .output()
22110                .expect("spawn git");
22111            assert!(out.status.success(), "git {args:?}");
22112        };
22113        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22114        g(&["add", "-A"]);
22115        g(&["commit", "-q", "-m", "fix(own): handle more"]);
22116        let second = posted(&mut f).await;
22117        let kept = f.runner.state.change_summary.clone().unwrap();
22118        assert_ne!(kept.head, first_head);
22119        assert!(second.body.contains("Not read yet."), "{}", second.body);
22120        assert!(!second.body.contains("so the first summary is read"));
22121        assert_eq!(second.title, "feat(own): other");
22122    }
22123
22124    #[tokio::test]
22125    async fn an_unavailable_summary_is_stated_and_never_invented() {
22126        let mut f = summary_fixture();
22127        std::fs::write(&f.reply, "no title line here").unwrap();
22128        let m = posted(&mut f).await;
22129        assert!(
22130            m.body
22131                .contains("No description of what this change does could be generated"),
22132            "{}",
22133            m.body
22134        );
22135        assert!(m.body.contains("internal reports remain in local records"));
22136        assert!(f.runner.state.change_summary.is_none());
22137        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
22138        // A preview asks nobody and claims nothing.
22139        let winner = f.runner.state.candidates[0].clone();
22140        let preview = f
22141            .runner
22142            .guarded_pr_message(&winner, None, false)
22143            .await
22144            .unwrap()
22145            .unwrap();
22146        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
22147    }
22148
22149    #[test]
22150    fn pr_message_review_single_commit_uses_its_subject() {
22151        let state = review_state(&["feat(nats): per-role user"]);
22152        let m = pr_message(&state, 'A');
22153        assert_eq!(m.title, "feat(nats): per-role user");
22154        assert!(!m.body.contains("Review the work already"), "{}", m.body);
22155        assert!(m.body.contains("## Commits under review"), "{}", m.body);
22156    }
22157
22158    #[test]
22159    fn pr_message_review_multi_commit_takes_the_oldest() {
22160        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
22161        let m = pr_message(&state, 'A');
22162        assert_eq!(m.title, "feat: the change");
22163        for s in ["feat: the change", "fix: typo", "fix: again"] {
22164            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
22165        }
22166    }
22167
22168    #[test]
22169    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
22170        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
22171            let mut state = review_state(&[first, "fix: later fixup"]);
22172            state.candidates[0].branch = "機能/ログイン".to_owned();
22173            let m = pr_message(&state, 'A');
22174            assert!(
22175                m.title.starts_with("chore: land candidate A of run"),
22176                "{}",
22177                m.title
22178            );
22179        }
22180    }
22181
22182    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22183        BranchFacts {
22184            commits: commits
22185                .iter()
22186                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22187                .collect(),
22188            stat: stat.to_owned(),
22189        }
22190    }
22191
22192    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22193        let mut r = review_round(true, 0, 1, 1, true, true);
22194        r.round = round;
22195        r.fix = notes.map(|n| FixRecord {
22196            agent: "fixer".to_owned(),
22197            addressed: Vec::new(),
22198            rejected: Vec::new(),
22199            notes: n.to_owned(),
22200            committed: true,
22201            failed: None,
22202            duration_ms: 0,
22203            continuation: None,
22204        });
22205        r
22206    }
22207
22208    #[test]
22209    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22210        let state = review_state(&["ignored"]);
22211        let f = facts(
22212            &[
22213                (
22214                    "fix(login): resolve PATH on macOS",
22215                    "Login shells skip rc files.",
22216                ),
22217                ("fix: address review", ""),
22218            ],
22219            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22220        );
22221        let m = pr_message_with(&state, 'A', Some(&f));
22222        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22223        assert!(
22224            m.body
22225                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
22226            "{}",
22227            m.body
22228        );
22229        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22230        assert!(m.body.contains("## Diff stat"), "{}", m.body);
22231        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22232        for banned in [
22233            "Review the work already",
22234            "no task statement",
22235            "Original task",
22236        ] {
22237            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22238        }
22239        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22240    }
22241
22242    #[test]
22243    fn pr_message_review_truncates_a_huge_first_commit_body() {
22244        let state = review_state(&["ignored"]);
22245        let f = facts(
22246            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22247            "s",
22248        );
22249        let m = pr_message_with(&state, 'A', Some(&f));
22250        assert!(m.body.len() < 30_000, "{}", m.body.len());
22251        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22252        assert!(
22253            m.body.contains("- ... 1 more commit(s)"),
22254            "{}",
22255            m.body.len()
22256        );
22257        assert!(m.body.ends_with("magi:candidate-a\n"));
22258    }
22259
22260    #[test]
22261    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22262        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22263        assert_eq!(m.title, "feat: x");
22264        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22265        assert!(!m.body.contains("Diff stat"), "{}", m.body);
22266        assert!(!m.body.contains("no task statement"), "{}", m.body);
22267    }
22268
22269    #[test]
22270    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22271        let mut state = review_state(&["日本語の件名"]);
22272        state.candidates[0].branch = "fix/macos-login-path".to_owned();
22273        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22274        state.candidates[0].branch = "機能/ログイン".to_owned();
22275        assert!(
22276            pr_message(&state, 'A')
22277                .title
22278                .starts_with("chore: land candidate A")
22279        );
22280    }
22281
22282    #[test]
22283    fn pr_message_review_fixes_survive_a_clean_final_round() {
22284        let mut state = review_state(&["feat: x"]);
22285        state.reviews = vec![
22286            round_with_notes(1, Some("handled the PATH case")),
22287            round_with_notes(2, None),
22288        ];
22289        let body = pr_message(&state, 'A').body;
22290        assert!(
22291            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22292            "{body}"
22293        );
22294        assert!(body.contains("### Round 1"), "{body}");
22295
22296        state.reviews = vec![
22297            round_with_notes(1, Some("first fix")),
22298            round_with_notes(2, Some("")),
22299            round_with_notes(3, Some("second fix")),
22300            round_with_notes(4, None),
22301        ];
22302        let body = pr_message(&state, 'A').body;
22303        assert!(
22304            body.contains("### Round 1") && !body.contains("first fix"),
22305            "{body}"
22306        );
22307        assert!(
22308            body.contains("### Round 3") && !body.contains("second fix"),
22309            "{body}"
22310        );
22311        assert!(body.contains("### Round 2"), "{body}");
22312    }
22313
22314    const FINAL: &str = "abcdef1234567890";
22315
22316    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22317        let mut r = round_with_notes(round, None);
22318        r.verified_head = verified.map(str::to_owned);
22319        r
22320    }
22321
22322    fn ok_gate(state: &mut RunState, head: Option<&str>) {
22323        state.gate = vec![CommandOutcome {
22324            command: "gate".to_owned(),
22325            code: Some(0),
22326            output_tail: String::new(),
22327            duration_ms: 0,
22328            resource_blocked: false,
22329        }];
22330        state.gate_ran = true;
22331        state.gate_head = head.map(str::to_owned);
22332    }
22333
22334    fn body_at(state: &RunState) -> String {
22335        pr_message_raw(
22336            state,
22337            'A',
22338            None,
22339            crate::pr_outcome::FinalHead::At(FINAL),
22340            ChangeNote::NotAsked,
22341        )
22342        .body
22343    }
22344
22345    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22346
22347    #[test]
22348    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22349        let mut state = review_state(&["feat: x"]);
22350        state.reviews = vec![
22351            round_with_notes(1, Some(SANDBOXED)),
22352            passing_round(2, Some(FINAL)),
22353        ];
22354        ok_gate(&mut state, Some(FINAL));
22355        let body = body_at(&state);
22356        assert!(!body.contains("Handled the PATH case."), "{body}");
22357        assert!(!body.contains("could not commit"), "{body}");
22358        assert!(!body.contains("could not run"), "{body}");
22359        assert!(
22360            body.contains("The fix is committed on the branch"),
22361            "{body}"
22362        );
22363        assert!(!body.contains("sandbox"), "{body}");
22364        assert!(
22365            body.contains("Tests (`verify.e2e`): passed on the final commit"),
22366            "{body}"
22367        );
22368        assert!(
22369            body.contains("Gate (`verify.gate`): passed on the final commit"),
22370            "{body}"
22371        );
22372        // The raw report is untouched in the record.
22373        assert!(
22374            state.reviews[0]
22375                .fix
22376                .as_ref()
22377                .unwrap()
22378                .notes
22379                .contains("could not commit")
22380        );
22381    }
22382
22383    #[test]
22384    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22385        let mut state = review_state(&["feat: x"]);
22386        let mut r1 = round_with_notes(1, Some(SANDBOXED));
22387        r1.fix.as_mut().unwrap().committed = false;
22388        let mut r2 = passing_round(2, Some(FINAL));
22389        r2.e2e[0].code = Some(1);
22390        state.reviews = vec![r1, r2];
22391        let body = body_at(&state);
22392        assert!(!body.contains("could not commit"), "{body}");
22393        assert!(!body.contains("could not run"), "{body}");
22394        assert!(body.contains("This round produced no commit"), "{body}");
22395        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22396        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22397        assert!(!body.contains("passed"), "{body}");
22398    }
22399
22400    #[test]
22401    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22402        let mut state = review_state(&["feat: x"]);
22403        state.reviews = vec![
22404            round_with_notes(1, Some(SANDBOXED)),
22405            passing_round(2, Some("0123456789abcdef")),
22406        ];
22407        ok_gate(&mut state, Some("0123456789abcdef"));
22408        let body = body_at(&state);
22409        assert!(!body.contains("HTTP tests could not run"), "{body}");
22410        // The commit claim is still settled by the structured record.
22411        assert!(!body.contains("could not commit"), "{body}");
22412        assert!(
22413            body.contains("passed on an earlier commit (`0123456`)"),
22414            "{body}"
22415        );
22416        assert!(!body.contains("passed on the final commit"), "{body}");
22417
22418        // A gate recorded before `gate_head` existed is "not recorded".
22419        ok_gate(&mut state, None);
22420        let body = body_at(&state);
22421        assert!(
22422            body.contains("passed on a commit that was not recorded"),
22423            "{body}"
22424        );
22425    }
22426
22427    #[test]
22428    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22429        let state = state_with_summary(
22430            "add retries to the client",
22431            "TITLE: feat: retries\n\nDid it.",
22432        );
22433        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22434        assert_eq!(m.title, "feat: retries");
22435        assert!(
22436            m.body.contains("<summary>Original task</summary>"),
22437            "{}",
22438            m.body
22439        );
22440        assert!(!m.body.contains("Commits under review"), "{}", m.body);
22441        assert!(
22442            m.body
22443                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22444            "{}",
22445            m.body
22446        );
22447    }
22448
22449    #[test]
22450    fn pr_message_review_bounds_a_long_english_subject() {
22451        let long = format!("feat: {}", "word ".repeat(100));
22452        let m = pr_message(&review_state(&[&long]), 'A');
22453        assert!(m.title.starts_with("feat: word"), "{}", m.title);
22454        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22455    }
22456
22457    #[test]
22458    fn pr_message_implementation_run_is_unchanged_by_review_support() {
22459        let state = state_with_summary("add retries\n\ndetails", "- did some things");
22460        let m = pr_message(&state, 'A');
22461        assert_eq!(m.title, "add retries");
22462        assert!(m.body.contains("<summary>Original task</summary>"));
22463        assert!(!m.body.contains("Commits under review"));
22464        assert_eq!(landing_subject_source(&state), state.instruction);
22465    }
22466
22467    #[test]
22468    fn review_run_squash_subject_is_the_change_not_the_prompt() {
22469        let state = review_state(&["feat: the change", "fix: typo"]);
22470        let source = landing_subject_source(&state);
22471        assert_eq!(land::merge_subject("", &source), "feat: the change");
22472        assert_eq!(
22473            land::merge_subject("magi: candidate A (uncommitted work)", &source),
22474            "feat: the change"
22475        );
22476        // An operator's rename still wins.
22477        assert_eq!(
22478            land::merge_subject("feat: renamed by hand", &source),
22479            "feat: renamed by hand"
22480        );
22481        let blank = review_state(&["日本語"]);
22482        assert!(
22483            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22484        );
22485    }
22486
22487    #[test]
22488    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22489        let state = review_state(&["feat: the change"]);
22490        let source = landing_subject_source(&state);
22491        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22492        assert_eq!(
22493            land::merge_subject(landing_title(&state, old), &source),
22494            "feat: the change"
22495        );
22496        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22497        let task = state_with_summary("add retries", "");
22498        assert_eq!(landing_title(&task, old), old);
22499    }
22500
22501    #[test]
22502    fn pr_message_describes_the_change_not_the_task() {
22503        let state = state_with_summary(
22504            "今回やってほしいこと: results projector を直す",
22505            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22506        );
22507        let m = pr_message(&state, 'A');
22508        assert_eq!(m.title, "fix(web): batch the runs list reads");
22509        assert!(
22510            m.body.starts_with("## Summary\n\n- reads run.json once"),
22511            "{}",
22512            m.body
22513        );
22514        assert!(!m.body.contains("TITLE:"), "{}", m.body);
22515        let task_at = m.body.find("今回やってほしいこと").unwrap();
22516        let details_at = m.body.find("<details>").unwrap();
22517        assert!(
22518            details_at < task_at,
22519            "the task lives inside <details>: {}",
22520            m.body
22521        );
22522        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22523        assert!(m.body.contains("magi:candidate-a"));
22524    }
22525
22526    #[test]
22527    fn pr_message_falls_back_to_the_task_without_a_title_line() {
22528        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22529        let m = pr_message(&state, 'A');
22530        assert_eq!(m.title, "add retries");
22531        assert!(
22532            m.body.contains("## Summary\n\n- did some things"),
22533            "{}",
22534            m.body
22535        );
22536
22537        let none = RunState::new(
22538            PathBuf::from("/repo"),
22539            "main".to_owned(),
22540            "abc1234".to_owned(),
22541            "add retries".to_owned(),
22542            Config::default(),
22543        );
22544        let m = pr_message(&none, 'A');
22545        assert_eq!(m.title, "add retries");
22546        assert!(!m.body.contains("## Summary"), "{}", m.body);
22547    }
22548
22549    #[test]
22550    fn pr_message_refuses_the_candidate_commit_subject() {
22551        for bad in [
22552            "TITLE: magi: candidate A (uncommitted work)",
22553            "TITLE: chore: stuff (uncommitted work)",
22554            "TITLE:   ",
22555        ] {
22556            let state = state_with_summary("add retries", bad);
22557            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22558        }
22559    }
22560
22561    #[test]
22562    fn pr_message_bounds_a_very_long_task_and_title() {
22563        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22564        let state = state_with_summary(&long, "- nothing");
22565        let m = pr_message(&state, 'A');
22566        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22567        assert!(!m.title.contains('\n'));
22568
22569        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22570        let m = pr_message(&state, 'A');
22571        assert!(m.title.starts_with("feat: "));
22572        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22573        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22574    }
22575
22576    fn long_title_of(instruction: &str) -> String {
22577        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22578    }
22579
22580    #[test]
22581    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22582        let first = "Make the landing path keep a readable title for long tasks";
22583        let line = format!(
22584            "{first}. {}",
22585            "And then keep going with more words ".repeat(20)
22586        );
22587        let t = long_title_of(&line);
22588        assert_eq!(t, first);
22589        assert!(!t.starts_with("chore: land"));
22590    }
22591
22592    #[test]
22593    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22594        let line = "word ".repeat(200);
22595        let t = long_title_of(&line);
22596        assert!(t.ends_with("word..."), "{t}");
22597        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22598    }
22599
22600    #[test]
22601    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22602        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22603            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22604        }
22605    }
22606
22607    #[test]
22608    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22609        let mut state = state_with_summary("再試行を追加する", "- nothing");
22610        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22611        state.config.graph.github_language = "ja".into();
22612        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22613    }
22614
22615    #[test]
22616    fn pr_message_title_limit_is_exact() {
22617        let at = "a".repeat(PR_TITLE_MAX);
22618        assert_eq!(long_title_of(&at), at);
22619        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22620        assert!(over.ends_with("..."), "{over}");
22621        assert_eq!(over.chars().count(), PR_TITLE_MAX);
22622    }
22623
22624    #[test]
22625    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22626        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22627        let t = long_title_of(&line);
22628        assert!(t.ends_with("..."), "{t}");
22629        assert!(t.is_ascii(), "{t}");
22630    }
22631
22632    #[test]
22633    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22634        let line = format!(
22635            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22636            "plus more filler words ".repeat(20)
22637        );
22638        let t = long_title_of(&line);
22639        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22640    }
22641
22642    #[test]
22643    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22644        let line = format!("\n\n# {}", "title words ".repeat(40));
22645        let state = state_with_summary(&line, "- nothing");
22646        let m = pr_message(&state, 'A');
22647        assert!(m.title.starts_with("title words"), "{}", m.title);
22648        assert_eq!(
22649            land::merge_subject(&m.title, &landing_subject_source(&state)),
22650            m.title
22651        );
22652        // An operator's rename wins untouched.
22653        assert_eq!(
22654            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22655            "feat: renamed by hand"
22656        );
22657    }
22658
22659    #[tokio::test]
22660    async fn github_text_rewrite_is_bounded_and_falls_back() {
22661        crate::run::pin_test_home();
22662        for (reply, accepted) in [
22663            (
22664                "TITLE: fix: retries\nAdd retries for failed requests.",
22665                true,
22666            ),
22667            (
22668                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22669                false,
22670            ),
22671            (
22672                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22673                true,
22674            ),
22675            (
22676                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22677                true,
22678            ),
22679            (
22680                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22681                true,
22682            ),
22683            (
22684                "TITLE: fix: retries\n日本語の説明です。 token=secret",
22685                false,
22686            ),
22687            ("TITLE: fix: retries\n", false),
22688        ] {
22689            let dir = tempfile::tempdir().unwrap();
22690            let mut runner = runner_at(RunStatus::Gating);
22691            runner.state = state_with_summary(
22692                "add retries",
22693                "TITLE: fix: retries\n日本語の説明を書きます。",
22694            );
22695            runner.state.repo = dir.path().to_owned();
22696            runner.state.candidates[0].worktree = dir.path().to_owned();
22697            let mut author = spec("alpha");
22698            author.command = vec![
22699                "sh".into(),
22700                "-c".into(),
22701                "printf '%s' \"$REWRITE_REPLY\"".into(),
22702            ];
22703            author.env.insert("REWRITE_REPLY".into(), reply.into());
22704            runner.state.config.agents = vec![author];
22705            seed_public_summary(&mut runner);
22706            let winner = runner.state.candidates[0].clone();
22707            let first = runner
22708                .guarded_pr_message(&winner, None, true)
22709                .await
22710                .unwrap();
22711            let message = if accepted {
22712                first.expect("a passing rewrite never asks")
22713            } else {
22714                // Withheld: the run parks on a question and posts nothing yet.
22715                assert!(first.is_none());
22716                assert!(runner.state.parked);
22717                let store = ask::Questions::open();
22718                let asked: Vec<_> = store
22719                    .list()
22720                    .into_iter()
22721                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22722                    .collect();
22723                assert_eq!(asked.len(), 1);
22724                assert!(!asked[0].detail.contains("secret"));
22725                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22726                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22727                assert_eq!(
22728                    asked[0]
22729                        .choices
22730                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
22731                    rewrite.is_some()
22732                );
22733                // Silence: the question lapses and the neutral text is used,
22734                // without a second rewrite call or a second question.
22735                store
22736                    .update(&asked[0].id, |q| {
22737                        q.abandon("test");
22738                        Ok(())
22739                    })
22740                    .unwrap();
22741                let again = runner
22742                    .guarded_pr_message(&winner, None, true)
22743                    .await
22744                    .unwrap()
22745                    .expect("an abandoned question falls back");
22746                let n = store
22747                    .list()
22748                    .into_iter()
22749                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22750                    .count();
22751                assert_eq!(n, 1, "asked once per rejected text");
22752                again
22753            };
22754            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22755            assert!(!message.body.contains("secret"));
22756            assert!(!message.body.contains("prefix"));
22757            assert!(!message.title.contains("secret"));
22758            assert_eq!(
22759                message.body.contains("Add retries for failed requests."),
22760                accepted
22761            );
22762            assert_eq!(
22763                runner.state.seats["github-text-rewrite"].turns, 1,
22764                "only one rewrite invocation"
22765            );
22766            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22767            if accepted {
22768                assert!(runner.state.github_text.is_none());
22769                assert!(!runner.state.parked);
22770                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22771                assert!(message.body.contains("## Final verification"));
22772                assert!(message.body.contains("Evidence: final diff"));
22773                assert!(
22774                    message
22775                        .body
22776                        .contains(&format!("magi:run/{}", runner.state.id))
22777                );
22778            }
22779            if !accepted {
22780                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22781                assert!(
22782                    message
22783                        .body
22784                        .contains(&format!("magi:run/{}", runner.state.id))
22785                );
22786            }
22787        }
22788    }
22789
22790    fn scripted(id: &str, reply: &str) -> AgentSpec {
22791        let mut a = spec(id);
22792        a.command = vec![
22793            "sh".into(),
22794            "-c".into(),
22795            "printf '%s' \"$REWRITE_REPLY\"".into(),
22796        ];
22797        a.env.insert("REWRITE_REPLY".into(), reply.into());
22798        a
22799    }
22800
22801    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22802    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22803
22804    /// Gate tests inject generated public output, not internal reports.
22805    fn seed_public_summary(runner: &mut Runner) {
22806        let repo = &runner.state.repo;
22807        let g = |args: &[&str]| {
22808            let out = std::process::Command::new("git")
22809                .args(args)
22810                .current_dir(repo)
22811                .quiet()
22812                .output()
22813                .unwrap();
22814            assert!(out.status.success(), "git {args:?}");
22815            String::from_utf8_lossy(&out.stdout).trim().to_owned()
22816        };
22817        g(&["init", "-q", "-b", "main"]);
22818        g(&[
22819            "-c",
22820            "user.name=Test",
22821            "-c",
22822            "user.email=test@example.invalid",
22823            "commit",
22824            "-q",
22825            "--allow-empty",
22826            "-m",
22827            "feat: fixture",
22828        ]);
22829        let head = g(&["rev-parse", "HEAD"]);
22830        g(&["update-ref", "refs/remotes/origin/main", &head]);
22831        let evidence = crate::pr_outcome::summary_evidence(
22832            &runner.state,
22833            crate::pr_outcome::FinalHead::At(&head),
22834        );
22835        let generated = &runner.state.candidates[0].summary;
22836        runner.state.change_summary = Some(crate::run::ChangeSummary {
22837            base: head.clone(),
22838            head,
22839            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22840            title: summary_title(generated).unwrap(),
22841            body: summary_without_title(generated),
22842        });
22843    }
22844
22845    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22846        let mut runner = runner_at(RunStatus::Gating);
22847        runner.state = state_with_summary("add retries", JA_SUMMARY);
22848        if review_only {
22849            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22850        }
22851        runner.state.repo = dir.to_owned();
22852        runner.state.candidates[0].worktree = dir.to_owned();
22853        runner.state.config.agents = agents;
22854        seed_public_summary(&mut runner);
22855        runner
22856    }
22857
22858    fn gate_questions(runner: &Runner) -> usize {
22859        ask::Questions::open()
22860            .list()
22861            .into_iter()
22862            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22863            .count()
22864    }
22865
22866    #[tokio::test]
22867    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22868        crate::run::pin_test_home();
22869        let dir = tempfile::tempdir().unwrap();
22870        let mut runner = rewrite_runner(
22871            dir.path(),
22872            true,
22873            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22874        );
22875        runner.state.config.roles.language_judge =
22876            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22877        let winner = runner.state.candidates[0].clone();
22878        let msg = runner
22879            .guarded_pr_message(&winner, None, true)
22880            .await
22881            .unwrap()
22882            .expect("an English rewrite posts without asking");
22883        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22884        assert!(msg.body.contains("<details>kept</details>"));
22885        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22886        assert_eq!(gate_questions(&runner), 0);
22887        assert!(runner.state.github_text.is_none());
22888        assert!(!runner.state.seats.contains_key("impl-A"));
22889        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22890    }
22891
22892    #[tokio::test]
22893    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22894        crate::run::pin_test_home();
22895        let dir = tempfile::tempdir().unwrap();
22896        let mut runner = rewrite_runner(
22897            dir.path(),
22898            false,
22899            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22900        );
22901        runner.state.config.roles.language_judge =
22902            Some(crate::config::AgentChoice::One("beta".into()));
22903        runner.state.seats.insert(
22904            "impl-A".into(),
22905            SeatState::new("impl-A", "alpha", runner.state.seed),
22906        );
22907        let winner = runner.state.candidates[0].clone();
22908        let msg = runner
22909            .guarded_pr_message(&winner, None, true)
22910            .await
22911            .unwrap()
22912            .expect("rewritten");
22913        assert!(msg.body.contains("<details>kept</details>"));
22914        assert_eq!(runner.state.seats["impl-A"].turns, 0);
22915        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22916        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22917    }
22918
22919    #[tokio::test]
22920    async fn rewrite_reuses_a_resumable_implementer_session() {
22921        crate::run::pin_test_home();
22922        let dir = tempfile::tempdir().unwrap();
22923        let mut runner = rewrite_runner(
22924            dir.path(),
22925            false,
22926            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22927        );
22928        runner.state.config.roles.language_judge =
22929            Some(crate::config::AgentChoice::One("beta".into()));
22930        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22931        seat.turns = 1;
22932        runner.state.seats.insert("impl-A".into(), seat);
22933        let winner = runner.state.candidates[0].clone();
22934        runner
22935            .guarded_pr_message(&winner, None, true)
22936            .await
22937            .unwrap()
22938            .expect("rewritten");
22939        assert_eq!(runner.state.seats["impl-A"].turns, 2);
22940        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22941    }
22942
22943    #[tokio::test]
22944    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22945        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
22946        // session, was out of quota, and nothing else was asked.
22947        crate::run::pin_test_home();
22948        let dir = tempfile::tempdir().unwrap();
22949        let mut exhausted = spec("alpha");
22950        exhausted.command = vec!["false".into()];
22951        let mut runner = rewrite_runner(
22952            dir.path(),
22953            false,
22954            vec![exhausted, scripted("beta", EN_REWRITE)],
22955        );
22956        // The implementer's own agent is in the role too: it is not asked twice.
22957        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22958            "alpha".into(),
22959            "beta".into(),
22960        ]));
22961        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22962        seat.turns = 1;
22963        runner.state.seats.insert("impl-A".into(), seat);
22964        let winner = runner.state.candidates[0].clone();
22965        let msg = runner
22966            .guarded_pr_message(&winner, None, true)
22967            .await
22968            .unwrap()
22969            .expect("the role rewrites after the implementer fails");
22970        assert!(msg.body.contains("`src/net.rs`"));
22971        assert_eq!(
22972            runner.state.seats["impl-A"].turns, 2,
22973            "the implementer was tried once"
22974        );
22975        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22976        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22977    }
22978
22979    #[tokio::test]
22980    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22981        crate::run::pin_test_home();
22982        let dir = tempfile::tempdir().unwrap();
22983        let mut broken = spec("bad");
22984        broken.command = vec!["false".into()];
22985        let mut runner = rewrite_runner(
22986            dir.path(),
22987            true,
22988            vec![
22989                broken,
22990                scripted("noformat", "Add retries."),
22991                scripted("good", EN_REWRITE),
22992            ],
22993        );
22994        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22995            "bad".into(),
22996            "noformat".into(),
22997            "good".into(),
22998        ]));
22999        let winner = runner.state.candidates[0].clone();
23000        let msg = runner
23001            .guarded_pr_message(&winner, None, true)
23002            .await
23003            .unwrap()
23004            .expect("the third agent rewrites");
23005        assert!(msg.body.contains("`src/net.rs`"));
23006        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
23007        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
23008    }
23009
23010    #[tokio::test]
23011    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
23012        crate::run::pin_test_home();
23013        // (agents, role): still Japanese; no TITLE; nothing resolvable.
23014        for (agents, role, saved) in [
23015            (
23016                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
23017                Some("beta"),
23018                true,
23019            ),
23020            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23021            (Vec::new(), None, false),
23022        ] {
23023            let dir = tempfile::tempdir().unwrap();
23024            let mut runner = rewrite_runner(dir.path(), true, agents);
23025            runner.state.config.roles.language_judge =
23026                role.map(|r| crate::config::AgentChoice::One(r.into()));
23027            let winner = runner.state.candidates[0].clone();
23028            let first = runner
23029                .guarded_pr_message(&winner, None, true)
23030                .await
23031                .unwrap();
23032            assert!(first.is_none(), "withheld text parks the run");
23033            assert!(runner.state.parked);
23034            assert_eq!(gate_questions(&runner), 1);
23035            let g = runner.state.github_text.as_ref().unwrap();
23036            assert_eq!(g.rewrite.is_some(), saved);
23037            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23038        }
23039    }
23040
23041    #[tokio::test]
23042    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23043        crate::run::pin_test_home();
23044        let dir = tempfile::tempdir().unwrap();
23045        let mut runner = runner_at(RunStatus::Gating);
23046        runner.state =
23047            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23048        runner.state.repo = dir.path().to_owned();
23049        runner.state.candidates[0].worktree = dir.path().to_owned();
23050        let mut author = spec("alpha");
23051        author.command = vec![
23052            "sh".into(),
23053            "-c".into(),
23054            "printf '%s' \"$REWRITE_REPLY\"".into(),
23055        ];
23056        author.env.insert(
23057            "REWRITE_REPLY".into(),
23058            format!(
23059                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23060                "English sentence. ".repeat(600)
23061            ),
23062        );
23063        runner.state.config.agents = vec![author];
23064        seed_public_summary(&mut runner);
23065        let winner = runner.state.candidates[0].clone();
23066        assert!(
23067            runner
23068                .guarded_pr_message(&winner, None, true)
23069                .await
23070                .unwrap()
23071                .is_none()
23072        );
23073        let record = runner.state.github_text.clone().unwrap();
23074        let rewrite = record.rewrite.clone().unwrap();
23075        assert_eq!(rewrite.failed, ["body-language"]);
23076        assert!(
23077            rewrite
23078                .body
23079                .ends_with(&format!("magi:run/{}", runner.state.id))
23080        );
23081        let store = ask::Questions::open();
23082        let q = store.get(record.question.as_ref().unwrap()).unwrap();
23083        assert!(
23084            q.choices
23085                .contains(&crate::github_text::USE_REWRITE.to_owned())
23086        );
23087        for label in [
23088            "ORIGINAL",
23089            "FALLBACK",
23090            "REWRITE",
23091            "Rules still failed: body-language",
23092            "truncated",
23093        ] {
23094            assert!(q.detail.contains(label), "{label}");
23095        }
23096        let artifact = std::fs::read_to_string(crate::run::artifact_path(
23097            &runner.state,
23098            &crate::github_text::artifact_name(&record.fingerprint),
23099        ))
23100        .unwrap();
23101        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23102        // Restart from serialized state, then retry after a bad owner title.
23103        runner.state =
23104            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23105        store
23106            .update(&q.id, |q| {
23107                q.say("まだ日本語")?;
23108                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23109            })
23110            .unwrap();
23111        assert!(
23112            runner
23113                .guarded_pr_message(&winner, None, true)
23114                .await
23115                .unwrap()
23116                .is_none()
23117        );
23118        let retry = runner.state.github_text.as_ref().unwrap();
23119        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
23120        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
23121        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
23122        assert!(
23123            q.posting_comparison
23124                .as_ref()
23125                .unwrap()
23126                .matches_detail(&q.detail)
23127        );
23128        store
23129            .update(&q.id, |q| {
23130                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23131            })
23132            .unwrap();
23133        let message = runner
23134            .guarded_pr_message(&winner, None, true)
23135            .await
23136            .unwrap()
23137            .unwrap();
23138        assert_eq!(message.title, rewrite.title);
23139        assert_eq!(message.body, rewrite.body);
23140        assert_eq!(
23141            runner.state.github_text.as_ref().unwrap().choice,
23142            Some(GithubTextChoice::Rewrite)
23143        );
23144        let again = runner
23145            .guarded_pr_message(&winner, None, true)
23146            .await
23147            .unwrap()
23148            .unwrap();
23149        assert_eq!(again.title, message.title);
23150        assert_eq!(again.body, message.body);
23151        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23152        assert!(
23153            !runner
23154                .state
23155                .events
23156                .iter()
23157                .any(|e| e.message.contains(&rewrite.title)
23158                    || e.message.contains("English sentence."))
23159        );
23160    }
23161
23162    #[tokio::test]
23163    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
23164        crate::run::pin_test_home();
23165        let dir = tempfile::tempdir().unwrap();
23166        let mut runner = runner_at(RunStatus::Gating);
23167        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23168        runner.state.repo = dir.path().to_owned();
23169        runner.state.candidates[0].worktree = dir.path().to_owned();
23170        seed_public_summary(&mut runner);
23171        let winner = runner.state.candidates[0].clone();
23172        assert!(
23173            runner
23174                .guarded_pr_message(&winner, None, true)
23175                .await
23176                .unwrap()
23177                .is_none()
23178        );
23179        let record = runner.state.github_text.as_ref().unwrap();
23180        assert!(record.rewrite.is_none());
23181        // Records written before the additive field deserialize with no rewrite.
23182        let mut old = serde_json::to_value(record).unwrap();
23183        old.as_object_mut().unwrap().remove("rewrite");
23184        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23185        assert!(old.rewrite.is_none());
23186        let id = record.question.clone().unwrap();
23187        ask::Questions::open()
23188            .update(&id, |q| {
23189                q.choices.push(crate::github_text::USE_REWRITE.into());
23190                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23191            })
23192            .unwrap();
23193        let message = runner
23194            .guarded_pr_message(&winner, None, true)
23195            .await
23196            .unwrap()
23197            .unwrap();
23198        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23199        assert_eq!(
23200            runner.state.github_text.as_ref().unwrap().choice,
23201            Some(GithubTextChoice::Fallback)
23202        );
23203    }
23204
23205    #[tokio::test]
23206    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23207        crate::run::pin_test_home();
23208        let dir = tempfile::tempdir().unwrap();
23209        let mut runner = runner_at(RunStatus::Gating);
23210        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23211        runner.state.repo = dir.path().to_owned();
23212        runner.state.candidates[0].worktree = dir.path().to_owned();
23213        seed_public_summary(&mut runner);
23214        let winner = runner.state.candidates[0].clone();
23215        assert!(
23216            runner
23217                .guarded_pr_message(&winner, None, true)
23218                .await
23219                .unwrap()
23220                .is_none()
23221        );
23222        let store = ask::Questions::open();
23223        let find = |run: &str| {
23224            store
23225                .list()
23226                .into_iter()
23227                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23228                .unwrap()
23229        };
23230        let run = runner.state.id.clone();
23231        let q = find(&run);
23232        let fp = runner
23233            .state
23234            .github_text
23235            .as_ref()
23236            .unwrap()
23237            .fingerprint
23238            .clone();
23239        let name = crate::github_text::artifact_name(&fp);
23240        let path = crate::run::artifact_path(&runner.state, &name);
23241        assert!(q.detail.contains(&path.display().to_string()));
23242        assert!(q.detail.contains("title-language: title"));
23243        let saved = std::fs::read_to_string(&path).expect("artifact written");
23244        assert!(saved.contains("日本語のタイトル"));
23245        assert!(q.cwd.is_none());
23246        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23247        // A bad replacement is refused and asked about once more.
23248        let say = |id: &str, text: &str, choice: &str| {
23249            store
23250                .update(id, |q| {
23251                    q.thread.push(ask::Turn {
23252                        who: ask::Who::Operator,
23253                        body: text.to_owned(),
23254                        at: jiff::Timestamp::now(),
23255                        note: None,
23256                    });
23257                    q.answer(ask::Answer::Choice(choice.to_owned()))
23258                })
23259                .unwrap();
23260        };
23261        say(&q.id, "まだ日本語", "use my text");
23262        assert!(
23263            runner
23264                .guarded_pr_message(&winner, None, true)
23265                .await
23266                .unwrap()
23267                .is_none()
23268        );
23269        let second = find(&run);
23270        assert_ne!(second.id, q.id);
23271        assert!(second.detail.contains("did not pass"));
23272        // The retry still compares the original text; the rejected
23273        // replacement is named by category only.
23274        assert!(second.detail.contains("日本語のタイトル"));
23275        assert!(!second.detail.contains("まだ日本語"));
23276        assert!(second.detail.contains(&path.display().to_string()));
23277        say(&second.id, "fix: retry failed requests", "use my text");
23278        let message = runner
23279            .guarded_pr_message(&winner, None, true)
23280            .await
23281            .unwrap()
23282            .expect("a vetted replacement is posted");
23283        assert_eq!(message.title, "fix: retry failed requests");
23284        assert_eq!(
23285            runner
23286                .state
23287                .github_text
23288                .as_ref()
23289                .unwrap()
23290                .chosen_title
23291                .as_deref(),
23292            Some("fix: retry failed requests")
23293        );
23294        assert!(runner.state.github_text.as_ref().unwrap().resolved);
23295        // Resumed again: the saved decision is applied, nothing is re-asked.
23296        let again = runner
23297            .guarded_pr_message(&winner, None, true)
23298            .await
23299            .unwrap()
23300            .unwrap();
23301        assert_eq!(again.title, message.title);
23302        assert_eq!(
23303            store
23304                .list()
23305                .into_iter()
23306                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23307                .count(),
23308            2
23309        );
23310    }
23311
23312    #[test]
23313    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23314        // What magi itself writes stays English under any configured language,
23315        // so a future localisation of these headings fails here. (The agents'
23316        // own text is also checked by the posting gate.)
23317        let mut state = state_with_summary(
23318            "add retries",
23319            "TITLE: fix(web): batch reads\n- reads run.json once",
23320        );
23321        state.config.graph.language = "ja".to_owned();
23322        let m = pr_message(&state, 'A');
23323        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23324        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23325
23326        // The task is the operator's own text: it goes in untouched, and the
23327        // fallback title (no summary) may be in its language too.
23328        let task = "今回やってほしいこと: results projector を直す";
23329        let mut state = state_with_summary(task, "- no title line");
23330        state.config.graph.language = "ja".to_owned();
23331        let m = pr_message(&state, 'A');
23332        assert_eq!(
23333            m.title,
23334            format!("chore: land candidate A of run {}", state.id)
23335        );
23336        assert!(
23337            m.body.contains(&format!(
23338                "<summary>Original task</summary>\n\n{task}\n\n</details>"
23339            )),
23340            "{}",
23341            m.body
23342        );
23343    }
23344
23345    #[test]
23346    fn pr_message_scrubs_home_paths_and_addresses() {
23347        let state = state_with_summary(
23348            "fix it in /Users/someone/src/x",
23349            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23350        );
23351        let m = pr_message(&state, 'A');
23352        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23353            assert!(!m.body.contains(leak), "{}", m.body);
23354        }
23355        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23356    }
23357
23358    #[test]
23359    fn pr_message_survives_a_task_that_closes_details() {
23360        let state = state_with_summary("a </details> b", "TITLE: fix: x");
23361        let m = pr_message(&state, 'A');
23362        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23363    }
23364
23365    #[test]
23366    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23367        let cmd = manual_merge_command(
23368            MergeStyle::Squash,
23369            Path::new("/repo"),
23370            "b",
23371            "fix: \"quoted\" $(x) `y`\n\nbody",
23372        );
23373        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23374    }
23375
23376    #[test]
23377    fn manual_merge_command_matches_the_configured_style() {
23378        let repo = Path::new("/repo");
23379        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23380
23381        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23382        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23383
23384        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23385        assert_eq!(
23386            squash,
23387            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23388             \"Merge magi run 0832 (candidate A)\""
23389        );
23390
23391        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23392        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23393    }
23394
23395    #[test]
23396    fn a_nudge_gets_a_quarter_of_the_budget() {
23397        // The judge and implement budgets magi ships with.
23398        assert_eq!(retry_budget(secs(1200), true), secs(300));
23399        assert_eq!(retry_budget(secs(3600), true), secs(900));
23400    }
23401
23402    #[test]
23403    fn a_resent_prompt_keeps_the_whole_budget() {
23404        // The seat kept no context, so the retry is the original job again and
23405        // shortening it would only guarantee a second failure.
23406        assert_eq!(retry_budget(secs(1200), false), secs(1200));
23407        assert_eq!(retry_budget(secs(60), false), secs(60));
23408    }
23409
23410    #[test]
23411    fn the_floor_never_exceeds_the_original_budget() {
23412        // A short configured timeout must not be *raised* by the floor: the
23413        // operator asked for a bound, and a retry may not outlast the attempt
23414        // it is retrying.
23415        assert_eq!(retry_budget(secs(60), true), secs(60));
23416        assert_eq!(retry_budget(secs(480), true), secs(120));
23417        assert_eq!(retry_budget(secs(0), true), secs(0));
23418    }
23419
23420    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23421        agent::CommandEvidence {
23422            id: "item1".to_owned(),
23423            description: "cargo test".to_owned(),
23424            exit_code,
23425            result_summary: String::new(),
23426            source: "codex".to_owned(),
23427        }
23428    }
23429
23430    #[test]
23431    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23432        // No evidence is not the same fact as unconfirmed evidence: a
23433        // backend with no adapter, or a reply that ran no commands at all,
23434        // must not be misread as carrying a dangling job.
23435        assert!(!has_unconfirmed_command(&[]));
23436    }
23437
23438    #[test]
23439    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23440        // Deliberately not a check on the exit code's *value*: a fixer
23441        // legitimately runs something that fails mid-iteration before it
23442        // succeeds, and that must never by itself reopen a valid report.
23443        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23444        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23445        assert!(!has_unconfirmed_command(&[
23446            evidence(Some(0)),
23447            evidence(Some(101))
23448        ]));
23449    }
23450
23451    #[test]
23452    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23453        assert!(has_unconfirmed_command(&[
23454            evidence(Some(0)),
23455            evidence(None)
23456        ]));
23457    }
23458
23459    #[test]
23460    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23461        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23462        assert_eq!(
23463            verified_noop_claim(true, &[], text).as_deref(),
23464            Some("already fixed by b32cfc4, on main.")
23465        );
23466    }
23467
23468    #[test]
23469    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23470        // A timeout or a bad exit code reads as the ordinary loss it is,
23471        // whatever the reply's own prose claims.
23472        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23473        assert!(verified_noop_claim(false, &[], text).is_none());
23474    }
23475
23476    #[test]
23477    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23478        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23479        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23480        // A confirmed command alongside the marker is fine.
23481        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23482    }
23483
23484    #[test]
23485    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23486        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23487    }
23488
23489    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
23490    /// pair, in order, labelled A, B, C, ...
23491    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23492        runner.state.candidates = shape
23493            .iter()
23494            .enumerate()
23495            .map(|(i, &(empty, verified))| Candidate {
23496                index: i,
23497                label: (b'A' + i as u8) as char,
23498                agent: "sonnet".to_owned(),
23499                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23500                worktree: PathBuf::from(format!("/wt/{i}")),
23501                summary: String::new(),
23502                stat: String::new(),
23503                files: 0,
23504                commits: 0,
23505                empty,
23506                failed: None,
23507                verified_noop: verified.map(str::to_owned),
23508                duration_ms: 0,
23509                folded: false,
23510            })
23511            .collect();
23512    }
23513
23514    #[test]
23515    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23516        ask_test_home();
23517        let mut runner = runner_at(RunStatus::Implementing);
23518        set_candidates(
23519            &mut runner,
23520            &[
23521                (true, Some("already on main at b32cfc4")),
23522                (true, Some("same fix, see the existing test")),
23523            ],
23524        );
23525
23526        runner
23527            .after_implement()
23528            .expect("a verified no-op is not an error");
23529
23530        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23531    }
23532
23533    #[test]
23534    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23535        ask_test_home();
23536        let mut runner = runner_at(RunStatus::Implementing);
23537        // Candidate A declares a verified no-op; candidate B simply wrote
23538        // nothing and said nothing about why. One candidate's claim is not
23539        // the whole run's agreement.
23540        set_candidates(
23541            &mut runner,
23542            &[(true, Some("already on main at b32cfc4")), (true, None)],
23543        );
23544
23545        let err = runner
23546            .after_implement()
23547            .expect_err("an unverified empty candidate must still fail the run");
23548
23549        assert!(
23550            err.to_string().contains("no candidate produced a change"),
23551            "{err}"
23552        );
23553        assert_eq!(runner.state.status, RunStatus::Failed);
23554    }
23555
23556    #[test]
23557    fn after_implement_still_fails_an_ordinary_all_empty_run() {
23558        ask_test_home();
23559        let mut runner = runner_at(RunStatus::Implementing);
23560        set_candidates(&mut runner, &[(true, None), (true, None)]);
23561
23562        let err = runner
23563            .after_implement()
23564            .expect_err("no candidate declared anything; this is an ordinary failure");
23565
23566        assert!(
23567            err.to_string().contains("no candidate produced a change"),
23568            "{err}"
23569        );
23570        assert_eq!(runner.state.status, RunStatus::Failed);
23571    }
23572}
23573
23574/// A graft rollback whose restore could not be confirmed (see
23575/// [`Runner::graft`]).
23576#[derive(Debug)]
23577struct GraftRestoreFailed(String);
23578
23579impl std::fmt::Display for GraftRestoreFailed {
23580    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23581        write!(
23582            f,
23583            "could not restore the winner to its pre-graft commit: {}",
23584            self.0
23585        )
23586    }
23587}
23588
23589impl std::error::Error for GraftRestoreFailed {}
23590
23591/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
23592/// not the agent: they never draw on an agent's own timeout, and running out of
23593/// them never reads as an agent failure.
23594#[cfg(test)]
23595mod resource_wait_tests {
23596    use super::*;
23597    use std::collections::BTreeMap;
23598
23599    fn sh(id: &str, script: &str) -> AgentSpec {
23600        AgentSpec {
23601            id: id.to_owned(),
23602            kind: crate::config::AgentKind::Command,
23603            model: None,
23604            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23605            extra_args: Vec::new(),
23606            env: BTreeMap::new(),
23607            prompt_delivery: None,
23608            max_turn_secs: None,
23609        }
23610    }
23611
23612    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23613        SeatJob {
23614            seat: SeatState::new("impl-A", &spec.id, 1),
23615            spec,
23616            cwd: dir.to_path_buf(),
23617            prompt: "do it".to_owned(),
23618            timeout,
23619            allow_write: true,
23620            sessions: false,
23621            artifacts: dir.join("art"),
23622            stem: "impl-A".to_owned(),
23623            handover: None,
23624        }
23625    }
23626
23627    fn state(resource_wait_secs: u64) -> RunState {
23628        let mut config = Config::default();
23629        config.graph.timeout_resource_wait = resource_wait_secs;
23630        RunState::new(
23631            PathBuf::from("/repo"),
23632            "main".to_owned(),
23633            "abc1234".to_owned(),
23634            "task".to_owned(),
23635            config,
23636        )
23637    }
23638
23639    fn hold(cache: &Path) -> crate::cache::Guard {
23640        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23641        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23642            crate::cache::AcquireOutcome::Acquired(g) => g,
23643            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23644        }
23645    }
23646
23647    async fn run(
23648        jobs: Vec<SeatJob>,
23649        sem: Arc<Semaphore>,
23650        cache: Option<&Path>,
23651        st: &mut RunState,
23652    ) -> Vec<(usize, SeatState, AgentOutcome)> {
23653        let prompts = Prompts::default();
23654        let ctx = WaveCtx {
23655            run: "r",
23656            node: "implement",
23657            prompts: &prompts,
23658            cache,
23659            round: None,
23660            carry_seats: false,
23661            pauses: None,
23662        };
23663        wave(jobs, sem, &ctx, st, 0).await
23664    }
23665
23666    #[tokio::test]
23667    async fn verification_cache_wait_preserves_the_command_budget() {
23668        crate::run::pin_test_home();
23669        let tmp = tempfile::tempdir().unwrap();
23670        let cache = tmp.path().join("target");
23671        let guard = hold(&cache);
23672        let release = tokio::spawn(async move {
23673            tokio::time::sleep(Duration::from_millis(100)).await;
23674            guard.release();
23675        });
23676        let mut st = state(10);
23677        let budget = Duration::from_millis(50);
23678        let wait = repository_wait(&st, None);
23679        let (outcomes, _) = with_cache_lease(
23680            &mut st,
23681            Some(&cache),
23682            "gate",
23683            "gate",
23684            tmp.path(),
23685            "abc1234",
23686            budget,
23687            "test gate",
23688            &wait,
23689            |_, available| async move {
23690                assert_eq!(
23691                    available, budget,
23692                    "contention must not shorten verification"
23693                );
23694                (Vec::new(), false, Vec::new())
23695            },
23696        )
23697        .await
23698        .unwrap();
23699        release.await.unwrap();
23700        assert!(outcomes.is_empty(), "{outcomes:?}");
23701        assert!(
23702            st.events
23703                .iter()
23704                .any(|event| event.message.contains("waiting for"))
23705        );
23706        assert!(st.resource_wait.is_none());
23707        // The verification body no longer owns the lease after returning.
23708        hold(&cache).release();
23709    }
23710
23711    #[tokio::test]
23712    async fn verification_cache_wait_expires_without_starting_commands() {
23713        crate::run::pin_test_home();
23714        let tmp = tempfile::tempdir().unwrap();
23715        let cache = tmp.path().join("target");
23716        let _held = hold(&cache);
23717        let mut st = state(0);
23718        let wait = repository_wait(&st, None);
23719        let (outcomes, _) = with_cache_lease(
23720            &mut st,
23721            Some(&cache),
23722            "gate",
23723            "gate",
23724            tmp.path(),
23725            "abc1234",
23726            Duration::from_secs(60),
23727            "test gate",
23728            &wait,
23729            |_, _| async { panic!("commands must not start without the cache lease") },
23730        )
23731        .await
23732        .unwrap();
23733        assert_eq!(outcomes.len(), 1);
23734        assert!(outcomes[0].resource_blocked);
23735        assert!(st.resource_wait.is_none());
23736        assert!(matches!(
23737            st.resource_block.unwrap().cause,
23738            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23739        ));
23740    }
23741
23742    #[tokio::test]
23743    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23744        crate::run::pin_test_home();
23745        let tmp = tempfile::tempdir().unwrap();
23746        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23747        let cache = tmp.path().join("cache");
23748        let mut state = state(21600);
23749        let wait = repository_wait(&state, None);
23750        let (outcomes, _) = with_cache_lease(
23751            &mut state,
23752            Some(&cache),
23753            "gate",
23754            "gate",
23755            tmp.path(),
23756            "head",
23757            Duration::from_secs(60),
23758            "gate",
23759            &wait,
23760            |_, _| async { panic!("freshness failure must not start verification") },
23761        )
23762        .await
23763        .unwrap();
23764        assert_eq!(outcomes.len(), 1);
23765        assert!(outcomes[0].resource_blocked);
23766        let block = state.resource_block.unwrap();
23767        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23768        assert!(!block.description().contains("timed out"));
23769    }
23770
23771    #[tokio::test]
23772    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23773        crate::run::pin_test_home();
23774        let tmp = tempfile::tempdir().unwrap();
23775        let cache = tmp.path().join("cache");
23776        let _held = hold(&cache);
23777        let mut state = state(21600);
23778        let pause = Pause::new();
23779        let interrupt = Pause::new();
23780        let cause = ParkCause::Operator;
23781        pause.park_for(cause.clone());
23782        let prompts = Prompts::default();
23783        let ctx = WaveCtx {
23784            run: "test",
23785            node: "fix",
23786            prompts: &prompts,
23787            cache: Some(&cache),
23788            round: None,
23789            carry_seats: false,
23790            pauses: Some((&pause, &interrupt)),
23791        };
23792        let result = wave(
23793            vec![job(
23794                sh("writer", "exit 99"),
23795                tmp.path(),
23796                Duration::from_secs(1),
23797            )],
23798            Arc::new(Semaphore::new(1)),
23799            &ctx,
23800            &mut state,
23801            0,
23802        )
23803        .await;
23804        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23805        assert_eq!(result[0].1.turns, 0);
23806        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23807        assert_eq!(
23808            state.resource_block.unwrap().cause,
23809            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23810        );
23811        assert!(state.active.is_empty());
23812    }
23813
23814    #[tokio::test]
23815    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23816        crate::run::pin_test_home();
23817        let tmp = tempfile::tempdir().unwrap();
23818        let cache = tmp.path().join("target-a");
23819        let guard = hold(&cache);
23820        // Held for longer than the agent's whole 1s budget.
23821        tokio::spawn(async move {
23822            tokio::time::sleep(Duration::from_millis(1500)).await;
23823            guard.release();
23824        });
23825        let mut st = state(60);
23826        let results = run(
23827            vec![job(
23828                sh("alpha", "echo ok"),
23829                tmp.path(),
23830                Duration::from_secs(1),
23831            )],
23832            Arc::new(Semaphore::new(1)),
23833            Some(&cache),
23834            &mut st,
23835        )
23836        .await;
23837        assert!(
23838            matches!(results[0].2, AgentOutcome::Ok(_)),
23839            "the wait must not be charged to the agent"
23840        );
23841        assert!(st.resource_block.is_none());
23842        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23843        assert!(st.resource_wait.is_none() && st.active.is_empty());
23844    }
23845
23846    #[tokio::test]
23847    #[cfg(unix)]
23848    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23849        use std::os::fd::AsRawFd as _;
23850        crate::run::pin_test_home();
23851        let tmp = tempfile::tempdir().unwrap();
23852        crate::git::git(tmp.path(), &["init"]).await.unwrap();
23853        let common = tmp.path().join(".git");
23854        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23855        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23856        assert_eq!(
23857            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23858            0
23859        );
23860        let mut st = state(1);
23861        st.config.coordination = crate::config::test_coordination();
23862        let mut reader = job(
23863            sh("reader", "echo must-not-run"),
23864            tmp.path(),
23865            Duration::from_secs(30),
23866        );
23867        reader.allow_write = false;
23868        let results = run(
23869            vec![reader.clone()],
23870            Arc::new(Semaphore::new(1)),
23871            None,
23872            &mut st,
23873        )
23874        .await;
23875        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23876        assert_eq!(results[0].1.turns, 0);
23877        assert!(FailClass::of(&results[0].2).is_none());
23878        assert_eq!(
23879            st.resource_block.as_ref().unwrap().kind,
23880            "repository coordination"
23881        );
23882        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23883        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23884        drop(external);
23885        let editor = crate::repo_lock::SeatLease::for_checkout(
23886            tmp.path(),
23887            true,
23888            crate::config::test_coordination().as_ref(),
23889        )
23890        .await
23891        .unwrap();
23892        let pause = Pause::new();
23893        let interrupt = Pause::new();
23894        let prompts = Prompts::default();
23895        let ctx = WaveCtx {
23896            run: "r",
23897            node: "review",
23898            prompts: &prompts,
23899            cache: None,
23900            round: None,
23901            carry_seats: false,
23902            pauses: Some((&pause, &interrupt)),
23903        };
23904        let (results, ()) = tokio::join!(
23905            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23906            async {
23907                tokio::time::sleep(Duration::from_millis(50)).await;
23908                pause.park();
23909            }
23910        );
23911        assert!(
23912            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23913        );
23914        assert_eq!(results[0].1.turns, 0);
23915        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23916        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23917        drop(editor);
23918    }
23919
23920    #[tokio::test]
23921    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23922        crate::run::pin_test_home();
23923        let tmp = tempfile::tempdir().unwrap();
23924        let cache = tmp.path().join("target-b");
23925        let _held = hold(&cache);
23926        let mut st = state(1);
23927        let results = run(
23928            vec![job(
23929                sh("alpha", "echo ok"),
23930                tmp.path(),
23931                Duration::from_secs(1),
23932            )],
23933            Arc::new(Semaphore::new(1)),
23934            Some(&cache),
23935            &mut st,
23936        )
23937        .await;
23938        let (_, seat, out) = &results[0];
23939        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23940        assert!(wave_blocked(&results));
23941        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23942        assert_eq!(seat.turns, 0, "nothing was asked");
23943        assert!(
23944            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23945            "no agent was spawned"
23946        );
23947        assert!(FailClass::of(out).is_none(), "not a failure class");
23948        let block = st.resource_block.as_ref().expect("recorded for the resume");
23949        assert_eq!(block.kind, "shared build cache");
23950        assert!(matches!(
23951            block.cause,
23952            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23953        ));
23954        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23955        assert!(st.active.is_empty());
23956    }
23957
23958    #[tokio::test]
23959    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23960        crate::run::pin_test_home();
23961        let tmp = tempfile::tempdir().unwrap();
23962        let mut st = state(1);
23963        let results = run(
23964            vec![job(
23965                sh("alpha", "echo ok"),
23966                tmp.path(),
23967                Duration::from_secs(1),
23968            )],
23969            Arc::new(Semaphore::new(0)),
23970            None,
23971            &mut st,
23972        )
23973        .await;
23974        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23975        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23976        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23977    }
23978
23979    #[tokio::test]
23980    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23981        crate::run::pin_test_home();
23982        let tmp = tempfile::tempdir().unwrap();
23983        let mut st = state(60);
23984        let results = run(
23985            vec![job(
23986                sh("alpha", "sleep 30"),
23987                tmp.path(),
23988                Duration::from_secs(1),
23989            )],
23990            Arc::new(Semaphore::new(1)),
23991            None,
23992            &mut st,
23993        )
23994        .await;
23995        let out = &results[0].2;
23996        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23997        let class = FailClass::of(out).expect("a failure");
23998        assert_eq!(class, FailClass::Timeout);
23999        assert!(should_hand_over(None, &class), "existing handover policy");
24000        assert!(st.resource_block.is_none());
24001    }
24002}