Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim shared by targeted repairs and repository reassignment.
534/// Held before asynchronous fix preflight and released on every return or panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549pub(crate) struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Capture task identity and repository assignment before the first
773        // save, so publication scope survives crashes and later task moves.
774        state.fetched_base = Some(fetched_base);
775        state.set_origin(origin)?;
776        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777        state.status = RunStatus::Prep;
778        state.event("start", format!("run {} created", state.id));
779        startup.finish(&mut state, Ok(()))?;
780        Ok(Self {
781            state,
782            roles,
783            sem: Arc::new(Semaphore::new(max_parallel)),
784            pause: Pause::new(),
785            interrupt: Pause::new(),
786        })
787    }
788
789    /// Open a review-only run against work that already exists on `branch`.
790    ///
791    /// The expensive half of the graph is the implement wave — measured at
792    /// 111 and 134 internal tool-loop turns on this repository, against a
793    /// handful for a judge or a reviewer. The cheap half is worth running on
794    /// hand-written work too, and there was no way to reach it.
795    ///
796    /// No new state and no schema change are needed: a run with **one** viable
797    /// candidate and a tally already decided degrades `execute` to exactly
798    /// review → gate → merge, because `judge` skips a single-candidate field,
799    /// `deliberate` has fewer than two first choices to reconcile, `vote`
800    /// returns early, `tally` is already present and `fold_losers` has no
801    /// losers. Resuming such a run therefore does the right thing as well.
802    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803        Self::review_taking_over(repo, branch, config, None, origin).await
804    }
805
806    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
807    /// at the same task still has `branch` checked out, its worktree is
808    /// released first if that is safe (see [`crate::handover`]), and the
809    /// review refuses with the reason if it is not. `None` is a hand-run
810    /// review: it has no earlier attempts, so only a worktree of a dead run
811    /// magi recorded itself can be released.
812    pub async fn review_taking_over(
813        repo: &Path,
814        branch: &str,
815        config: Config,
816        takeover: Option<crate::handover::Takeover>,
817        origin: Origin,
818    ) -> Result<Self> {
819        let repo = git::toplevel(repo).await?;
820        let missing = agent::missing_programs(&config.agents);
821        if !missing.is_empty() {
822            bail!(
823                "these agent programs are not on PATH: {}. Fix the roster in \
824                 magi.toml or install them.",
825                missing.join(", ")
826            );
827        }
828        let base_branch =
829            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830                .await?;
831        if base_branch == branch {
832            bail!("`{branch}` is the base branch; there is nothing to review against");
833        }
834        let fetched_base = resolve_base(
835            &repo,
836            &base_branch,
837            &config.merge.remote,
838            config.coordination.as_ref(),
839        )
840        .await?;
841        let base_commit = fetched_base.commit.clone();
842
843        let roles = config.resolve_roles()?;
844        let max_parallel = config.graph.max_parallel.max(1);
845        let mut state = RunState::new(
846            repo.clone(),
847            base_branch,
848            base_commit.clone(),
849            format!("Review the work already on branch `{branch}`."),
850            config,
851        );
852        state.fetched_base = Some(fetched_base);
853        state.set_origin(origin)?;
854        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856        let opened = async {
857            startup.stage(&mut state, "handover")?;
858
859            // Released before anything else touches the branch: a stale local
860            // branch is moved with `git branch -f`, which git refuses while an
861            // earlier attempt's worktree still has it checked out. Everything
862            // after this point that can fail puts the old run back.
863            // A hand-run review has no task, hence no earlier attempts, but a
864            // worktree of a dead run magi made may still be released.
865            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866                earlier: Vec::new(),
867                home: crate::run::home(),
868                choice: None,
869                task_instruction: None,
870            });
871            let wait = repository_wait(&state, None);
872            let coordination_config = state.config.coordination.clone();
873            let coordination =
874                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875                    .await?;
876            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877            if let Some(released) = &released {
878                state.event(
879                    "release",
880                    format!(
881                        "took `{branch}` over from run {}: its worktree was released: {}",
882                        crate::run::short_of(&released.old_id),
883                        released.audit
884                    ),
885                );
886            }
887            let opened = async {
888                startup.stage(&mut state, "review-preflight")?;
889                // The owner's answer to an earlier divergence question is applied
890                // here: after the release (git will not move a checked-out branch)
891                // and before the sync that would otherwise ask again.
892                state.foreign_checkout = crate::handover::preflight_review(
893                    &repo,
894                    branch,
895                    &takeover.home,
896                    &state.config.merge.remote,
897                )
898                .await?;
899                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901                }
902                if let Some(choice) = takeover.choice.as_ref()
903                    && let Err(e) =
904                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905                            .await
906                {
907                    return Err(e.context("applying the owner's answer about the diverged branch"));
908                }
909                // An earlier attempt of this task that published the task head, and
910                // whose work is the branch under review, hands that head over: the
911                // review run updates the same pull request. Without one (a legacy
912                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
913                // own head and its own open pull request is adopted as before.
914                for id in takeover.earlier.iter().rev() {
915                    if let Ok(old) = RunState::load_under(id, &takeover.home)
916                        && let Some(head) = old.pr_head.clone()
917                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918                    {
919                        state.pr_head = Some(head);
920                        break;
921                    }
922                }
923                drop(coordination);
924                startup.stage(&mut state, "open-review")?;
925                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926            }
927            .await;
928            if opened.is_err()
929                && let Some(released) = &released
930            {
931                // Restoration is ownership checked and best effort. Keep the original
932                // startup cause even if reacquiring coordination is no longer possible.
933                match crate::repo_lock::SeatLease::operation_with(
934                    &repo,
935                    coordination_config.as_ref(),
936                    &wait,
937                )
938                .await
939                {
940                    Ok(_coordination) => {
941                        released.restore(&repo, branch).await;
942                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
943                            .map(|old| {
944                                format!(
945                                    "predecessor released_to={:?}; candidate path exists={}",
946                                    old.released_to,
947                                    old.candidates.iter().any(|c| {
948                                        c.branch == branch && c.worktree.exists()
949                                    }),
950                                )
951                            })
952                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954                    }
955                    Err(error) => {
956                        tracing::warn!(
957                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958                            state.id
959                        );
960                        state.event(
961                            "startup-restore",
962                            format!("restoration not attempted: {error:#}"),
963                        );
964                    }
965                }
966            }
967            opened
968        }
969        .await;
970        startup.finish(&mut state, opened)?;
971        Ok(Self {
972            state,
973            roles,
974            sem: Arc::new(Semaphore::new(max_parallel)),
975            pause: Pause::new(),
976            interrupt: Pause::new(),
977        })
978    }
979
980    /// Recover exactly the operator-selected public implementation, without touching old sources.
981    pub async fn recover_pr(
982        repo: &Path,
983        task: &str,
984        selection: crate::pr_recovery::Selection,
985        mut config: Config,
986    ) -> Result<Self> {
987        selection.configure(&mut config)?;
988        let assignment = crate::run::task_assignment_for(task)?;
989        selection.check_task_assignment(task, &selection.instruction, assignment)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.set_origin(crate::run::Origin::queue(task))?;
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256            Err(e) => Err(e),
1257        };
1258        if let Err(e) = setup {
1259            git::worktree_remove_clean(repo, &worktree).await.ok();
1260            return Err(e);
1261        }
1262        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263            .await
1264            .unwrap_or(0);
1265        if commits == 0 {
1266            git::worktree_remove_clean(repo, &worktree).await.ok();
1267            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268        }
1269        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270            .await
1271            .map(|f| f.len())
1272            .unwrap_or(0);
1273        if files == 0
1274            && let (Ok(head_tree), Ok(base_tree)) = (
1275                git::tree_of(&worktree, "HEAD").await,
1276                git::tree_of(&worktree, &base_commit).await,
1277            )
1278            && head_tree == base_tree
1279        {
1280            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281            git::worktree_remove_clean(repo, &worktree).await.ok();
1282            bail!(
1283                "`{branch}` at {} has a tree identical to base {}; this usually means \
1284                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285                 against `{}/{branch}`) rather than an empty change",
1286                short(&head),
1287                short(&base_commit),
1288                state.config.merge.remote
1289            );
1290        }
1291        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292            .await
1293            .unwrap_or_default();
1294
1295        state.candidates.push(Candidate {
1296            index: 0,
1297            label: 'A',
1298            // Not an agent id on purpose: nothing in the roster wrote this, and
1299            // the stats tables must not credit anyone with a win for it.
1300            agent: EXISTING_BRANCH.to_owned(),
1301            branch: review_branch,
1302            worktree,
1303            summary: String::new(),
1304            stat,
1305            files,
1306            commits,
1307            empty: false,
1308            failed: None,
1309            verified_noop: None,
1310            duration_ms: 0,
1311            folded: false,
1312        });
1313        state.tally = Some(Tally {
1314            first_choice: BTreeMap::from([('A', 0)]),
1315            borda: BTreeMap::new(),
1316            winner: 'A',
1317            rankings: 0,
1318            unanimous_initial: false,
1319            deliberated: false,
1320            changed_votes: 0,
1321            unanimous_final: false,
1322            tie_break: None,
1323            // No panel sat, so no quorum applies. Zero judges is the correct
1324            // number for work that never competed, and must not be reported as
1325            // a collapsed panel.
1326            judges: 0,
1327            present: 0,
1328            quorum: 0,
1329            met_quorum: true,
1330            uncontested: Some("review-only run: nothing competed".to_owned()),
1331        });
1332        state.status = RunStatus::Reviewing;
1333        state.prep_complete = Some(true);
1334        state.event(
1335            "start",
1336            format!(
1337                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338                state.id
1339            ),
1340        );
1341        Ok(())
1342    }
1343
1344    /// Authorize one bounded synchronization interval, under the caller's queue task
1345    /// claim and the same owner/execution locks used by run maintenance.
1346    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347        Self::authorize_base_recovery_request(home, id, task, false, None).await
1348    }
1349
1350    /// Queue release uses its durable approval generation exactly once.
1351    pub(crate) async fn authorize_base_recovery_from_resumption(
1352        home: &Path,
1353        id: &str,
1354        task: &str,
1355    ) -> Result<()> {
1356        Self::authorize_base_recovery_request(home, id, task, true, None).await
1357    }
1358
1359    pub(crate) async fn authorize_base_recovery_control(
1360        home: &Path,
1361        task: &str,
1362        control: &mut crate::base_recovery_control::Transaction,
1363    ) -> Result<()> {
1364        let id = control.request.run.clone();
1365        Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1366    }
1367
1368    async fn authorize_base_recovery_request(
1369        home: &Path,
1370        id: &str,
1371        task: &str,
1372        resumption: bool,
1373        mut control: Option<&mut crate::base_recovery_control::Transaction>,
1374    ) -> Result<()> {
1375        let _owner = crate::pr_head::owner_lock(home, id)?;
1376        let mut state = RunState::load_under(id, home)?;
1377        crate::pr_head::ensure_owner(&state)?;
1378        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1379        anyhow::ensure!(
1380            queued.execution_runs().last().is_some_and(|run| run == id)
1381                && queued.status != crate::queue::TaskStatus::Done
1382                && !queued.fresh_start
1383                && queued.review_branch.is_none(),
1384            "run task ownership changed"
1385        );
1386        anyhow::ensure!(
1387            state
1388                .origin
1389                .as_ref()
1390                .and_then(|o| o.task.as_deref())
1391                .is_none_or(|owner| owner == task),
1392            "run belongs to another task"
1393        );
1394        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1395        anyhow::ensure!(
1396            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1397            "run is live or its ownership is unknown"
1398        );
1399        // Judge the previous driver's lease before acquiring our maintenance
1400        // lease. The owner lock excludes new driver claims across both steps.
1401        let _execution = crate::driver::maintenance(home, id)?;
1402        anyhow::ensure!(
1403            state.status.resumable()
1404                && state.released_to.is_none()
1405                && state.retired_to.is_none()
1406                && state.pr_taken_over_by.is_none(),
1407            "run ownership is final"
1408        );
1409        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1410        anyhow::ensure!(
1411            !state.pr_publication_pending
1412                && !crate::handover::landing_checkpoint(&state)
1413                && state.merge_decision.is_none(),
1414            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1415        );
1416        let winner = state.winner().context("run has no retained winner")?;
1417        anyhow::ensure!(
1418            git::is_clean(&winner.worktree).await?,
1419            "retained worktree has uncommitted changes"
1420        );
1421        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1422        anyhow::ensure!(
1423            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1424            "retained worktree does not match the winner branch"
1425        );
1426        if let Some(control) = control.as_ref() {
1427            anyhow::ensure!(
1428                head.starts_with(&control.request.head),
1429                "retained HEAD changed since the owner request"
1430            );
1431            anyhow::ensure!(
1432                state
1433                    .base_sync_recovery
1434                    .stop
1435                    .as_ref()
1436                    .is_none_or(|stop| stop.head == head),
1437                "retained HEAD differs from the recorded stop head; inspect it first"
1438            );
1439        }
1440        state.ensure_base_episode();
1441        state.record_base_sync_stop(&head);
1442        state.save_under(home)?;
1443        let fetched =
1444            git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1445        anyhow::ensure!(
1446            fetched.ok(),
1447            "cannot authorize base recovery without a fetched base"
1448        );
1449        let approved_tip = git::rev_parse(
1450            &state.repo,
1451            &format!("{}/{}", state.config.merge.remote, state.base_branch),
1452        )
1453        .await?;
1454        if let Some(control) = control.as_deref_mut() {
1455            let queue = crate::queue::Queue::at(home.join("queue"));
1456            control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1457            if let Some(prior) = state
1458                .base_sync_recovery
1459                .authorizations
1460                .iter()
1461                .find(|a| a.request.as_deref() == Some(&control.key()))
1462            {
1463                anyhow::ensure!(
1464                    !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1465                    "recovery request was consumed or invalidated; it cannot queue again"
1466                );
1467                return control.enqueue(&queue);
1468            }
1469        }
1470        if resumption {
1471            anyhow::ensure!(
1472                crate::daemon::compatible_run_instruction(&state, &queued),
1473                "resumption instruction does not match the retained run"
1474            );
1475            let approval = queued
1476                .resumption
1477                .as_ref()
1478                .context("no operator resumption")?;
1479            anyhow::ensure!(
1480                approval.instruction == queued.instruction,
1481                "resumption approval does not match the current task instruction"
1482            );
1483            state.authorize_base_sync_resumption(task, approval)?;
1484        } else {
1485            state.authorize_base_sync(task)?;
1486        }
1487        let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1488        state
1489            .base_sync_recovery
1490            .authorizations
1491            .last_mut()
1492            .unwrap()
1493            .scope = Some(crate::base_sync_recovery::Scope {
1494            tip: approved_tip,
1495            instruction: queued.instruction.clone(),
1496            episode,
1497        });
1498        if let Some(control) = control.as_deref_mut() {
1499            let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1500            authorization.request = Some(control.key());
1501            authorization.generation = control.generation();
1502            authorization.decision = control.decision();
1503            authorization.question = control
1504                .request
1505                .decision
1506                .as_ref()
1507                .map(|_| control.request.request.clone());
1508        }
1509        state.event(
1510            "base-recovery",
1511            "owner authorized a bounded base synchronization interval; fresh verification required",
1512        );
1513        state.save_under(home)?;
1514        if let Some(control) = control {
1515            control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1516        }
1517        Ok(())
1518    }
1519
1520    /// Reopen an existing run.
1521    pub fn resume(id: &str) -> Result<Self> {
1522        let state = RunState::load(id)?;
1523        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1524        if state.startup.as_ref().is_some_and(|s| !s.complete)
1525            || (state.status == RunStatus::Prep
1526                && state.instruction.trim().is_empty()
1527                && state.candidates.is_empty()
1528                && state.prep_complete == Some(false))
1529        {
1530            bail!(
1531                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1532            );
1533        }
1534        crate::pr_head::ensure_owner(&state)?;
1535        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1536            bail!(
1537                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1538            );
1539        }
1540        if let Some(to) = &state.released_to {
1541            bail!(
1542                "run {} cannot be resumed: its worktree was released to run {}",
1543                state.short(),
1544                crate::run::short_of(to)
1545            );
1546        }
1547        let roles = state.config.resolve_roles()?;
1548        if let Some(id) = &state.config.graph.selected_implementer
1549            && state.candidates.iter().any(|c| c.agent != *id)
1550        {
1551            bail!(
1552                "replacement implementer pin does not match the saved candidates; task must stay held"
1553            );
1554        }
1555        let max_parallel = state.config.graph.max_parallel.max(1);
1556        Ok(Self {
1557            state,
1558            roles,
1559            sem: Arc::new(Semaphore::new(max_parallel)),
1560            pause: Pause::new(),
1561            interrupt: Pause::new(),
1562        })
1563    }
1564
1565    /// Walk the graph to a terminal state, skipping nodes already recorded.
1566    ///
1567    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1568    /// the phone - ends here, so this is the one place a run that ended
1569    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1570    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1571    pub async fn execute(&mut self) -> Result<()> {
1572        anyhow::ensure!(
1573            self.state.startup.as_ref().is_none_or(|s| s.complete),
1574            "startup incomplete; inspect retained evidence before explicit recovery"
1575        );
1576        if let Some(selection) = &self.state.pr_recovery
1577            && (selection.source_proof.is_some() || selection.source_run.is_some())
1578        {
1579            Box::pin(selection.verify_source(&self.state.repo)).await?;
1580        }
1581        // Queue claims cover daemon work; this also serializes manual drivers
1582        // consuming a repair answer, including the new review and decision.
1583        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1584            || self.state.land_approval.is_some()
1585            || self.state.merge_decision.is_some()
1586            || !self.state.merge_repairs.is_empty()
1587            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1588        {
1589            let claim = FixClaim::acquire(&self.state.dir())?;
1590            crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1591            let disk = RunState::load(&self.state.id)?;
1592            let checkpoint = |s: &RunState| {
1593                (
1594                    s.merge_repairs.len(),
1595                    s.merge_repairs
1596                        .last()
1597                        .map(|r| (r.question.clone(), r.phase)),
1598                    s.land_approval
1599                        .as_ref()
1600                        .map(|a| (a.question.clone(), a.head.clone())),
1601                )
1602            };
1603            if checkpoint(&disk) != checkpoint(&self.state) {
1604                bail!(
1605                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1606                );
1607            }
1608            Some(claim)
1609        } else {
1610            None
1611        };
1612        let mut execution = self.claim_driver().await?;
1613        let runner = &mut *execution.runner;
1614        let wait = runner.repository_wait();
1615        // Keep the graph's large future off the callers' stack, including
1616        // standalone task attempts running on a default-sized test thread.
1617        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1618        let result = match result {
1619            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1620                record_repository_block(&mut runner.state, &e);
1621                runner.park_here()?;
1622                Ok(())
1623            }
1624            other => other,
1625        };
1626        execution.driver.finish(&mut runner.state).await?;
1627        let ended = if result.is_err() {
1628            Some({
1629                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1630                if let Err(error) = &result {
1631                    notice.message = format!(
1632                        "Run {} stopped with an error: {}",
1633                        runner.state.id,
1634                        error.to_string().lines().next().unwrap_or("unknown error")
1635                    );
1636                }
1637                notice
1638            })
1639        } else {
1640            crate::notices::run_ended(&runner.state)
1641        };
1642        if let Some(notice) = ended {
1643            crate::notices::raise(notice);
1644        }
1645        result
1646    }
1647
1648    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1649        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1650    }
1651
1652    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1653        let wait = self.repository_wait();
1654        coordinate_operation(&mut self.state, cwd, &wait).await
1655    }
1656
1657    /// Validate ownership and persist the driver claim in one critical section.
1658    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1659        let owner = match crate::pr_head::owner_lock_wait(
1660            &crate::run::home(),
1661            &self.state.id,
1662            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1663        )
1664        .await
1665        {
1666            Ok(owner) => owner,
1667            Err(error) => {
1668                let detail = format!("worktree maintenance or ownership update: {error:#}");
1669                // Admission did not own run.json: retain this resource outcome
1670                // in memory for queue settlement without overwriting a concurrent
1671                // retirement or publication transaction.
1672                self.state.resource_block = Some(crate::run::ResourceBlock {
1673                    cause: crate::run::ResourceBlockCause::Other,
1674                    node: "start".into(),
1675                    kind: "worktree maintenance".into(),
1676                    detail: detail.clone(),
1677                    at: Timestamp::now(),
1678                });
1679                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1680            }
1681        };
1682        let disk = RunState::load(&self.state.id)?;
1683        crate::pr_head::ensure_owner(&disk)?;
1684        let claimed =
1685            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1686        // Only proven-live ownership refuses here. Unknown records (legacy runs
1687        // with no claim marker or process identity) fall through to the
1688        // exclusive execution lease taken just below, which is what actually
1689        // keeps a second driver out.
1690        anyhow::ensure!(
1691            disk.liveness(claimed) != crate::run::Liveness::Live,
1692            "run already has a live driver; wait for its execution to end"
1693        );
1694        if let Some(to) = &disk.released_to {
1695            bail!(
1696                "run {} cannot continue: its worktree was released to run {}",
1697                self.state.short(),
1698                crate::run::short_of(to)
1699            );
1700        }
1701        anyhow::ensure!(
1702            serde_json::to_value(&disk.base_sync_recovery)?
1703                == serde_json::to_value(&self.state.base_sync_recovery)?,
1704            "base recovery changed since this run was loaded; reload the saved run"
1705        );
1706        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1707        let mut execution = DriverExecution {
1708            runner: self,
1709            driver,
1710        };
1711        // Declare after the execution guard: on any claim error this short
1712        // lock must drop before that guard reloads and releases ownership.
1713        let _owner = owner;
1714        crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1715            .await?;
1716        let state = &mut execution.runner.state;
1717        // Moving again, so it is no longer parked. Set before the walk rather
1718        // than in `resume`, so every way of re-entering the graph clears it
1719        // and a card cannot claim a run is waiting to be resumed while the
1720        // agents are already working.
1721        state.parked = false;
1722        state.resource_wait = None;
1723        state.resource_block = None;
1724        // Any seat this state still lists as answering belongs to whatever
1725        // process last drove this run — this one included, if it crashed
1726        // mid-wave. Cleared and flushed immediately, before anything else
1727        // runs, so a resume can never show a seat as live when nothing is
1728        // asking it anything yet; the node that actually dispatches the next
1729        // wave repopulates it.
1730        state.clear_active();
1731        // Recorded in the same spot, and flushed together with the clear
1732        // above: this is the pid a reader checks (`RunState::liveness`) when
1733        // no daemon claim exists to answer "is a process still driving this
1734        // run" — a plain `magi run` / `magi review` typed into a terminal
1735        // claims nothing there. Always overwritten, never only-if-absent, so
1736        // a resumed run's stale pid from a previous, possibly-dead process
1737        // can never survive into this one's own report. Unlike
1738        // `clear_active`, this changes on every single `execute()` call, so
1739        // the save below is now unconditional rather than only-if-cleared.
1740        //
1741        // `driver_started_at` is recorded in the same breath, from this same
1742        // pid, so `liveness` can tell a live pid that is genuinely still us
1743        // apart from one the OS has since handed to an unrelated process —
1744        // see that field's own doc for why the pid alone is not enough.
1745        let pid = std::process::id();
1746        state.driver_pid = Some(pid);
1747        state.driver_started_at = crate::proc::process_started_at(pid);
1748        state.driver_exited = false;
1749        state.driver_claimed = Some(true);
1750        state.begin_pass();
1751        execution.driver.bind(state);
1752        state.save()?;
1753        execution.driver.record(state)?;
1754        Ok(execution)
1755    }
1756
1757    /// No conflict checkpoint proves dispatch was never reached. End that
1758    /// consumed generation so an owner can explicitly authorize a fresh one.
1759    async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1760        let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1761        if let Err(error) = &result
1762            && !error.is::<crate::repo_lock::ResourceBlocked>()
1763            && self
1764                .state
1765                .repair_grants
1766                .last()
1767                .is_some_and(|g| g.conflict.is_none())
1768        {
1769            self.state.repair_grants.last_mut().unwrap().phase =
1770                crate::repair_budget::Phase::Complete;
1771            self.state.status = RunStatus::Blocked;
1772            self.state.save()?;
1773            bail!(
1774                "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1775                self.state.passes.last().map_or(0, |p| p.index)
1776            );
1777        }
1778        result
1779    }
1780
1781    /// A grant owns only these repair attempts. Confirmation reviews cannot
1782    /// create another fixer allowance, and every operation is restart-checkpointed.
1783    async fn execute_granted_repair(&mut self) -> Result<()> {
1784        use crate::repair_budget::{Kind, Phase};
1785        let index = self.state.repair_grants.len() - 1;
1786        let winner = self
1787            .state
1788            .winner()
1789            .cloned()
1790            .context("repair winner missing")?;
1791        loop {
1792            let grant = self.state.repair_grants[index].clone();
1793            match grant.phase {
1794                Phase::Pending => {
1795                    if grant.spent >= grant.receipt.request.rounds {
1796                        self.state.repair_grants[index].phase = Phase::Complete;
1797                        self.state.status = RunStatus::Blocked;
1798                        self.state.save()?;
1799                        return Ok(());
1800                    }
1801                    // Reserve before dispatch. A crash can consume an attempt,
1802                    // but can never duplicate it or reset saved fixer history.
1803                    self.state.repair_grants[index].spent += 1;
1804                    self.state.repair_grants[index].phase = Phase::Fixing;
1805                    self.state.save()?;
1806                    match grant.receipt.request.kind {
1807                        Kind::BaseConflict => self.repair_base_conflict(true).await?,
1808                        Kind::Gate => {
1809                            let failed = self.state.gate.clone();
1810                            let _ = self.gate_fix_round(&winner, &failed).await?;
1811                        }
1812                        Kind::Review => {
1813                            let mut record = self
1814                                .state
1815                                .reviews
1816                                .last()
1817                                .cloned()
1818                                .context("review failure missing")?;
1819                            let round = record.round;
1820                            let step = self.run_fix_round(&winner, &mut record).await?;
1821                            self.state.reviews[round - 1] = record;
1822                            if matches!(step, FixStep::Recorded { report_lost: true }) {
1823                                self.state.repair_grants[index].phase = Phase::Complete;
1824                                self.state.status = RunStatus::Blocked;
1825                                self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1826                                self.state.save()?;
1827                                return Ok(());
1828                            }
1829                            if matches!(step, FixStep::Resource) {
1830                                self.stop_for_resource()?;
1831                                return Ok(());
1832                            }
1833                            self.state.save()?;
1834                        }
1835                    }
1836                    if self.resource_halt()? {
1837                        return Ok(());
1838                    }
1839                }
1840                Phase::Fixing => {
1841                    if grant.receipt.request.kind == Kind::BaseConflict {
1842                        self.repair_base_conflict(false).await?;
1843                    }
1844                    // Interrupted fixers may have left source changes or an
1845                    // unpushed commit. Preserve them before fresh assessment.
1846                    let _coordination = self.coordinate(&winner.worktree).await?;
1847                    let rescued = git::rescue_commit(
1848                        &winner.worktree,
1849                        "magi: preserve interrupted granted repair",
1850                    )
1851                    .await?;
1852                    self.state.note_withheld("repair-grant", &rescued.withheld);
1853                    // Fresh checks must supersede even a same-head failure.
1854                    // Archive rather than erase failure evidence or counters.
1855                    self.state
1856                        .verification_history
1857                        .push(crate::run::VerificationEvidence {
1858                            superseded_by: None,
1859                            archived_at: Some(Timestamp::now()),
1860                            fixes_spent: Some(self.state.gate_fixes.len()),
1861                            gate_head: self.state.gate_head.take(),
1862                            gate_ran: std::mem::take(&mut self.state.gate_ran),
1863                            gate: std::mem::take(&mut self.state.gate),
1864                            e2e: self.state.head_e2e.take(),
1865                        });
1866                    if let Some(e2e) = self.state.gate_repair_e2e.take() {
1867                        self.state
1868                            .verification_history
1869                            .push(crate::run::VerificationEvidence {
1870                                superseded_by: None,
1871                                archived_at: Some(Timestamp::now()),
1872                                fixes_spent: Some(self.state.gate_fixes.len()),
1873                                gate_head: None,
1874                                gate_ran: false,
1875                                gate: Vec::new(),
1876                                e2e: Some(e2e),
1877                            });
1878                    }
1879                    self.state.repair_grants[index].review_start = self.state.reviews.len();
1880                    self.state.repair_grants[index].head =
1881                        git::rev_parse(&winner.worktree, "HEAD").await?;
1882                    self.state.repair_grants[index].phase = Phase::Reviewing;
1883                    if grant.receipt.request.kind == Kind::BaseConflict {
1884                        self.state.base_sync_recovery.stop = None;
1885                        if let Some(sync) = &mut self.state.base_sync {
1886                            sync.tip = grant
1887                                .receipt
1888                                .request
1889                                .tip
1890                                .clone()
1891                                .context("repair base tip missing")?;
1892                            sync.conflict = None;
1893                            sync.behind = 0;
1894                        }
1895                        refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1896                    }
1897                    self.state.save()?;
1898                }
1899                Phase::Reviewing => {
1900                    self.review_loop_pass().await?;
1901                    if self.resource_halt()? {
1902                        return Ok(());
1903                    }
1904                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1905                    let clean = self.state.reviews.len() > grant.review_start
1906                        && self.state.reviews.last().is_some_and(|r| {
1907                            r.head == head
1908                                && r.clean
1909                                && r.expected > 0
1910                                && r.answered == r.expected
1911                                && matches!(
1912                                    r.e2e_status(),
1913                                    E2eStatus::Passed | E2eStatus::NotConfigured
1914                                )
1915                        });
1916                    if !clean {
1917                        // A confirmation review carries no additional fix budget.
1918                        self.state.repair_grants[index].phase = if grant.receipt.request.kind
1919                            == Kind::Review
1920                            && grant.spent < grant.receipt.request.rounds
1921                            && self.state.reviews.last().is_some_and(|r| {
1922                                r.fix.is_none()
1923                                    && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1924                            }) {
1925                            Phase::Pending
1926                        } else {
1927                            Phase::Complete
1928                        };
1929                        self.state.status = RunStatus::Blocked;
1930                        self.state.event(
1931                            "repair-grant",
1932                            "fresh review did not clear the repaired head; no publication",
1933                        );
1934                        self.state.save()?;
1935                        if self.state.repair_grants[index].phase == Phase::Pending {
1936                            continue;
1937                        }
1938                        return Ok(());
1939                    }
1940                    self.state.repair_grants[index].phase = Phase::Gating;
1941                    self.state.save()?;
1942                }
1943                Phase::Gating => {
1944                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1945                    self.state.reconcile_verification_head(&head);
1946                    let wait = self.repository_wait();
1947                    if self
1948                        .state
1949                        .reviews
1950                        .last()
1951                        .is_none_or(|r| r.head != head || !r.clean)
1952                    {
1953                        self.state.status = RunStatus::Blocked;
1954                        self.state.repair_grants[index].phase = Phase::Complete;
1955                        self.state.event(
1956                            "repair-grant",
1957                            "head changed since fresh review; repair held",
1958                        );
1959                        self.state.save()?;
1960                        return Ok(());
1961                    }
1962                    if !verify_current_head(&mut self.state, &winner, &wait).await? {
1963                        return Ok(());
1964                    }
1965                    let outcomes = self.run_gate(&winner).await?;
1966                    if verify_inconclusive(&outcomes) {
1967                        return Ok(());
1968                    }
1969                    let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1970                    if after != head {
1971                        self.state.repair_grants[index].review_start = self.state.reviews.len();
1972                        self.state.repair_grants[index].head = after;
1973                        self.state.repair_grants[index].phase = Phase::Reviewing;
1974                        self.state.status = RunStatus::Blocked;
1975                        self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1976                        self.state.save()?;
1977                        return Ok(());
1978                    }
1979                    let passed = outcomes.iter().all(CommandOutcome::ok);
1980                    // Follow-up grants owe the same guarded checkout delivery
1981                    // as the conflict repair that introduced the new head.
1982                    if passed
1983                        && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1984                    {
1985                        return Ok(());
1986                    }
1987                    self.state.repair_grants[index].succeeded = passed;
1988                    self.state.gate = outcomes;
1989                    self.state.gate_ran = true;
1990                    self.state.status = if passed {
1991                        RunStatus::Gating
1992                    } else {
1993                        RunStatus::Blocked
1994                    };
1995                    self.state.repair_grants[index].phase = if !passed
1996                        && grant.receipt.request.kind == Kind::Gate
1997                        && grant.spent < grant.receipt.request.rounds
1998                    {
1999                        Phase::Pending
2000                    } else {
2001                        Phase::Complete
2002                    };
2003                    if passed && self.state.config.merge.mode == MergeMode::Pr {
2004                        // Persist publication debt with the successful checkpoint.
2005                        self.state.pr_publication_pending = true;
2006                    } else if passed {
2007                        self.state.status = RunStatus::Ready;
2008                    }
2009                    self.state.save()?;
2010                    if !passed {
2011                        continue;
2012                    }
2013                    // Existing PR ownership and push lease stay in place. The
2014                    // normal publisher owes the new reviewed/gated SHA, and
2015                    // land's fresh owner approval remains mandatory.
2016                    if self.state.config.merge.mode != MergeMode::Pr {
2017                        self.state.status = RunStatus::Ready;
2018                        self.state.event(
2019                            "repair-grant",
2020                            "repair verified locally; grant carries no merge authorization",
2021                        );
2022                        self.state.save()?;
2023                        return Ok(());
2024                    }
2025                    return self.merge().await;
2026                }
2027                Phase::Complete => return Ok(()),
2028            }
2029        }
2030    }
2031
2032    async fn execute_graph(&mut self) -> Result<()> {
2033        // Even a terminal run may reenter through an explicit recovery action.
2034        // Restore before prep shortcuts, saved conversations or graph nodes use
2035        // a disposable path; GC may already have removed its original objects.
2036        let archive_home = crate::run::home().canonicalize()?;
2037        if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2038            let repo = self.state.repo.clone();
2039            let _coordination = self.coordinate(&repo).await?;
2040            ensure_setup_config(&mut self.state, &repo).await?;
2041            crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
2042        }
2043        self.state.configure_recovery_resume()?;
2044        if self
2045            .state
2046            .repair_grants
2047            .last()
2048            .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2049        {
2050            return self.execute_granted_repair().await;
2051        }
2052        if self
2053            .state
2054            .repair_grants
2055            .last()
2056            .is_some_and(|g| !g.succeeded)
2057        {
2058            self.state.status = RunStatus::Blocked;
2059            self.state.save()?;
2060            return Ok(());
2061        }
2062        // Preservation may invalidate an old empty classification. Resolve it
2063        // before any resume shortcut can reuse a verdict or publish a branch.
2064        self.persist_candidates().await?;
2065        // A saved base stop outranks stage markers, including legacy Voting.
2066        // Recovery never reenters implementation, deliberation or voting.
2067        if self.state.base_sync_blocked() {
2068            self.sync_to_base().await?;
2069            if !self.state.base_sync_blocked()
2070                && self.state.status != RunStatus::AlreadyInBase
2071                && !self.resource_halt()?
2072            {
2073                self.finish_after_tally().await?;
2074            }
2075            return Ok(());
2076        }
2077        if self
2078            .state
2079            .merge_decision
2080            .as_ref()
2081            .is_some_and(|decision| !decision.closed)
2082            && self.state.status != RunStatus::Landing
2083        {
2084            if crate::merge_decision::resumable_hold(&self.state) {
2085                self.state.merge = Some(MergeOutcome {
2086                    mode: MergeMode::Pr,
2087                    ok: false,
2088                    detail: self
2089                        .state
2090                        .merge_decision
2091                        .as_ref()
2092                        .unwrap()
2093                        .target
2094                        .pr
2095                        .clone(),
2096                    empty: false,
2097                });
2098                self.run_land().await?;
2099                self.settle_questions();
2100                return Ok(());
2101            }
2102            bail!(
2103                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2104            );
2105        }
2106        if self.state.config.graph.selected_implementer.is_some()
2107            && self.state.status == RunStatus::Blocked
2108            && self.state.tally.is_none()
2109            && self.state.candidate_preservation.is_empty()
2110            && self.state.interrupted_implementations.is_empty()
2111        {
2112            bail!(
2113                "selected implementation is held; explicitly release the owning task to retry it"
2114            );
2115        }
2116
2117        if self.state.merge_repairs.last().is_some_and(|r| {
2118            matches!(
2119                r.phase,
2120                crate::run::MergeRepairPhase::Requested
2121                    | crate::run::MergeRepairPhase::Fixing
2122                    | crate::run::MergeRepairPhase::Reviewing
2123            )
2124        }) {
2125            if self.repair_merge_review().await? {
2126                self.run_land().await?;
2127            }
2128            self.settle_questions();
2129            return Ok(());
2130        }
2131        // Publication already passed review and gate. Resume only that node:
2132        // syncing to a newer base here could change the verified commit.
2133        if self.state.pr_publication_pending {
2134            self.merge().await?;
2135            if self.resource_halt()? {
2136                return Ok(());
2137            }
2138            self.state.save()?;
2139            self.settle_questions();
2140            return Ok(());
2141        }
2142        // A run that already lost its quorum never resumes into the verdict
2143        // machinery: `deliberate` and `vote` would otherwise clobber the
2144        // stalled marker back to Voting and the run would keep going past a
2145        // verdict that is no longer trustworthy. Everything already recorded is
2146        // kept, so the run stays resumable (or foldable) for a human to pick up.
2147        //
2148        // On --resume the run gets one chance to repair itself: the seats a
2149        // rate limit took out are re-asked. If their quota has since reset and
2150        // the quorum is restored, the run picks up and finishes; otherwise it
2151        // stays stale and still-resumable for a later retry. If it does not
2152        // recover, the returned status stays `Stalled` and nothing was
2153        // clobbered (the recovery only mutates entries for the lost seats).
2154        if self.state.status == RunStatus::Stalled {
2155            if self.recover_stall().await? {
2156                self.finish_after_tally().await?;
2157            } else {
2158                // Still below quorum: persist the marker and stay resumable.
2159                self.state.save()?;
2160            }
2161            return Ok(());
2162        }
2163        // A run parked inside `land` - watching CI, mid fix-round, or
2164        // waiting on the owner's merge approval - resumes directly into it,
2165        // never back through `prep`. Everything before `merge` already
2166        // concluded; that is the only way `status` reaches `Landing` in the
2167        // first place. Re-walking `review_loop` first would also be actively
2168        // wrong: its own status recomputation (see its doc) treats any
2169        // clean round as reason to set `status` to `Gating`, which would
2170        // clobber this marker before `merge` ever ran, and this run would
2171        // never find its way back into `land` at all.
2172        if self.state.status == RunStatus::Landing
2173            || crate::handover::landing_checkpoint(&self.state)
2174        {
2175            self.state.land_sync_pending = true;
2176            self.state.save()?;
2177            self.run_land().await?;
2178            // `run_land` may have settled the run right here - CI came back
2179            // green and the PR merged, say - without ever passing back
2180            // through `merge`'s own trailing call. Whatever it left `status`
2181            // as is what this has to read.
2182            self.settle_questions();
2183            return Ok(());
2184        }
2185        // A run parked at `merge` on the owner's word about a withheld PR text
2186        // resumes straight into `merge`. Re-walking the tail would sync to the
2187        // base again (possibly rebasing, possibly spending the fixer) after the
2188        // gate had passed, and then push a tree nobody reviewed or gated.
2189        if self.state.github_text.is_some()
2190            && self.state.merge.is_none()
2191            && self.state.status == RunStatus::Gating
2192        {
2193            self.merge().await?;
2194            if self.resource_halt()? {
2195                return Ok(());
2196            }
2197            self.state.save()?;
2198            return Ok(());
2199        }
2200        if let Err(e) = self.prep().await {
2201            if e.is::<crate::repo_lock::ResourceBlocked>() {
2202                return Err(e);
2203            }
2204            self.state.status = RunStatus::Blocked;
2205            self.state
2206                .prep_failure
2207                .get_or_insert_with(|| format!("{e:#}"));
2208            self.state
2209                .event("prep", format!("preparation stopped: {e:#}"));
2210            self.state.save()?;
2211            return Err(e);
2212        }
2213        if self.park_here()? {
2214            return Ok(());
2215        }
2216        self.advise().await?;
2217        if self.park_here()? {
2218            return Ok(());
2219        }
2220        self.implement().await?;
2221        if self.park_here()? {
2222            return Ok(());
2223        }
2224        // `after_implement` already saved the state and settled any open
2225        // questions when it set this; nothing later in the graph has
2226        // anything to judge.
2227        if self.state.status == RunStatus::VerifiedNoop {
2228            return Ok(());
2229        }
2230        self.judge().await?;
2231        if self.park_here()? {
2232            return Ok(());
2233        }
2234        self.deliberate().await?;
2235        if self.park_here()? {
2236            return Ok(());
2237        }
2238        self.vote().await?;
2239        if self.park_here()? {
2240            return Ok(());
2241        }
2242        self.tally()?;
2243        // A verdict that lost its quorum is not trustworthy: do not review,
2244        // gate, or merge on it. Everything already done is kept, so the run
2245        // stays resumable (or foldable); the human can replace the agent that
2246        // ran out of quota and pick it up.
2247        if self.state.status == RunStatus::Stalled {
2248            // Persist the stalled marker now — the normal end-of-execute save
2249            // below is below this early return, and without it a resumed run
2250            // would reload a pre-tally status and keep going.
2251            self.state.save()?;
2252            return Ok(());
2253        }
2254        self.finish_after_tally().await?;
2255        Ok(())
2256    }
2257
2258    /// A wave could not start its agents because a shared resource stayed busy
2259    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
2260    /// resumable. Unstarted seats have no result; completed peers retain
2261    /// their saved candidate state.
2262    fn stop_for_resource(&mut self) -> Result<()> {
2263        // Resource acquisition observes cancellation inside a node. Complete
2264        // the same durable park as every other node boundary before returning.
2265        if self.pause.parked() || self.interrupt.parked() {
2266            self.park_here()?;
2267            return Ok(());
2268        }
2269        if self.state.resource_block.is_some() {
2270            self.state.status = RunStatus::Blocked;
2271        }
2272        self.state.save()
2273    }
2274
2275    /// Stop walking the graph if a node ended on a resource block.
2276    fn resource_halt(&mut self) -> Result<bool> {
2277        if self.state.resource_block.is_none() {
2278            return Ok(false);
2279        }
2280        self.stop_for_resource()?;
2281        Ok(true)
2282    }
2283
2284    /// Park here if asked to, recording it in the run's own timeline.
2285    ///
2286    /// Returns whether the caller should stop walking the graph. The state is
2287    /// saved either way by the node that just finished; this adds the event so
2288    /// the operator's card says why a run that is neither finished nor moving
2289    /// is sitting where it is.
2290    fn park_here(&mut self) -> Result<bool> {
2291        let parking = self.pause.parked() || self.interrupt.parked();
2292        if !parking && self.state.resource_block.is_some() {
2293            self.stop_for_resource()?;
2294            return Ok(true);
2295        }
2296        // Either handle asking is enough - see `Pause`'s own doc for why
2297        // they are never the same one. `interrupt` is checked second so a
2298        // reason it carries is preferred in the message below over a plain
2299        // shutdown park racing it at the same boundary.
2300        if !self.pause.parked() && !self.interrupt.parked() {
2301            return Ok(false);
2302        }
2303        if self.state.parked {
2304            return Ok(true);
2305        }
2306        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2307        if let Some(block) = &mut self.state.resource_block
2308            && matches!(
2309                block.cause,
2310                crate::run::ResourceBlockCause::Cancelled { .. }
2311            )
2312        {
2313            block.cause = crate::run::ResourceBlockCause::Cancelled {
2314                park: cause.clone(),
2315            };
2316        }
2317        let why = match cause.as_ref().map(ParkCause::label) {
2318            Some(reason) => format!(
2319                "parked after `{}` ({reason}) — resume to carry on from here",
2320                self.state.status.as_str()
2321            ),
2322            None => format!(
2323                "parked after `{}` — resume to carry on from here",
2324                self.state.status.as_str()
2325            ),
2326        };
2327        self.state.event("park", why);
2328        self.state
2329            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2330        self.state.parked = true;
2331        self.state.save()?;
2332        Ok(true)
2333    }
2334
2335    /// Only a new explicit operator approval may retry failed preparation.
2336    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2337        if self.state.prep_failure.is_none() {
2338            return Ok(());
2339        }
2340        let failed_at = self
2341            .state
2342            .events
2343            .iter()
2344            .rev()
2345            .find(|event| event.node == "prep")
2346            .map_or(self.state.updated_at, |event| event.at);
2347        anyhow::ensure!(
2348            approval_at > failed_at,
2349            "preparation retry requires a new explicit release after the failure"
2350        );
2351        self.state.prep_failure = None;
2352        self.state.prep_complete = Some(false);
2353        self.state
2354            .event("prep-retry", "operator approved retry of preparation");
2355        Ok(())
2356    }
2357
2358    /// Hand the runner the pause `magi serve`'s own shutdown watches.
2359    pub fn on_pause(&mut self, pause: Pause) {
2360        self.pause = pause;
2361    }
2362
2363    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
2364    /// scheduler asking this one run - and no other - to park so a task
2365    /// marked [`crate::queue::Task::interrupt`] can run alone. See
2366    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
2367    /// handle.
2368    pub fn watch_interrupt(&mut self, pause: Pause) {
2369        self.interrupt = pause;
2370    }
2371
2372    /// Abandon this run's own open questions, once `status` has actually
2373    /// settled rather than merely paused.
2374    ///
2375    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
2376    /// either back up with the candidates, the review round and the seat
2377    /// sessions already on disk, so a question an implementer asked mid-round
2378    /// may still get a real answer read by a real resume. Only the statuses
2379    /// `resumable` excludes are actually final: the run merged, it reached
2380    /// `Ready` with nothing left to do, it failed outright with no
2381    /// established point to continue from, or every candidate agreed, with
2382    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
2383    /// every one of those the seat that asked is gone for good, exactly like
2384    /// the run being deleted under `magi run rm` - so the same cleanup
2385    /// applies, worded for what actually happened instead of "the run was
2386    /// deleted".
2387    ///
2388    /// Best-effort and silent on success: called from every place `status`
2389    /// can land on one of those three, including ones a resumed run revisits,
2390    /// so it must cost nothing when there was nothing open to begin with.
2391    fn settle_questions(&mut self) {
2392        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2393            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2394        }
2395    }
2396
2397    /// The tail of the graph after a trustworthy tally: fold losers, review,
2398    /// gate, merge, and persist.
2399    async fn finish_after_tally(&mut self) -> Result<()> {
2400        // Before the losers are folded: their branches are what the graft
2401        // reads from.
2402        self.graft().await?;
2403        if self.resource_halt()? {
2404            return Ok(());
2405        }
2406        self.fold_losers().await?;
2407        // Before review starts, and again right before the gate: a run's
2408        // review rounds can themselves take long enough for the base to move
2409        // a second time, and the gate is the one node whose "green" gets
2410        // acted on.
2411        self.sync_to_base().await?;
2412        if self.hold_base_sync().await? || self.resource_halt()? {
2413            return Ok(());
2414        }
2415        if self.state.status == RunStatus::AlreadyInBase {
2416            return Ok(());
2417        }
2418        self.review_loop().await?;
2419        if self.resource_halt()? {
2420            return Ok(());
2421        }
2422        self.sync_to_base().await?;
2423        if self.hold_base_sync().await? || self.resource_halt()? {
2424            return Ok(());
2425        }
2426        if self.state.status == RunStatus::AlreadyInBase {
2427            return Ok(());
2428        }
2429        self.gate().await?;
2430        self.merge().await?;
2431        if self.resource_halt()? {
2432            return Ok(());
2433        }
2434        self.state.save()?;
2435        Ok(())
2436    }
2437
2438    // ---------------------------------------------------------------- prep
2439
2440    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2441    /// run with the step and its output in the event; it is never swallowed.
2442    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2443    /// resume starts over instead of finding candidates and missing seats.
2444    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2445        let result = match ensure_setup_config(&mut self.state, repo).await {
2446            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2447            Err(e) => Err(e),
2448        };
2449        if let Err(e) = result {
2450            git::worktree_remove_clean(repo, wt).await.ok();
2451            if in_prep {
2452                self.rollback_prep(repo).await;
2453            }
2454            self.state.status = RunStatus::Blocked;
2455            self.state.event(
2456                "setup",
2457                format!("worktree setup failed in {}: {e:#}", wt.display()),
2458            );
2459            self.state.save()?;
2460            return Err(e);
2461        }
2462        Ok(())
2463    }
2464
2465    /// Undo everything `prep` created, so a resume re-runs it whole.
2466    async fn rollback_prep(&mut self, repo: &Path) {
2467        let root = self.state.worktree_root();
2468        for c in std::mem::take(&mut self.state.candidates) {
2469            if matches!(
2470                git::worktree_remove_clean(repo, &c.worktree).await,
2471                Ok(true)
2472            ) {
2473                git::branch_delete(repo, &c.branch).await.ok();
2474            } else {
2475                self.state.candidates.push(c);
2476            }
2477        }
2478        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2479        for j in 1..=seats {
2480            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2481                .await
2482                .ok();
2483        }
2484        for k in 1..=self.state.config.graph.advisors {
2485            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2486                .await
2487                .ok();
2488        }
2489    }
2490
2491    async fn prep(&mut self) -> Result<()> {
2492        if let Some(reason) = &self.state.prep_failure {
2493            bail!(
2494                "candidate preparation previously failed: {reason}; correct the source before retrying"
2495            );
2496        }
2497        if self
2498            .state
2499            .prep_complete
2500            .unwrap_or(!self.state.candidates.is_empty())
2501        {
2502            return Ok(());
2503        }
2504        let repo = self.state.repo.clone();
2505        // Includes rollback, config, refs, checkout creation and setup. No
2506        // repository mutation may precede the real shared lock acquisition.
2507        let _coordination = self.coordinate(&repo).await?;
2508        let base = self.state.base_commit.clone();
2509        let plan = self.state.sources.plan(&repo, &base).await?;
2510        self.rollback_prep(&repo).await;
2511        if !self.state.candidates.is_empty() {
2512            bail!(
2513                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2514            );
2515        }
2516        self.state.prep_complete = Some(false);
2517        self.state.status = RunStatus::Prep;
2518        self.state.save()?;
2519        let repo = self.state.repo.clone();
2520        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2521        let root = self.state.worktree_root();
2522        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2523
2524        // The hook is the write-time half of the blindness contract; the
2525        // presentation filter in `blind` is the half that cannot be bypassed.
2526        let hooks_dir = self.state.dir().join("hooks");
2527        if self.state.config.blind.commit_msg_hook {
2528            std::fs::create_dir_all(&hooks_dir)
2529                .with_context(|| format!("create {}", hooks_dir.display()))?;
2530            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2531            let path = hooks_dir.join("commit-msg");
2532            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2533            make_executable(&path)?;
2534            // Ref-counted rather than a plain idempotent set: with more than
2535            // one run able to be in flight in the same repository at once
2536            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2537            // true?" check cannot tell "another run of mine still needs
2538            // this" from "nobody does", and the run that happens to finish
2539            // first would disable the hook out from under a sibling still
2540            // relying on it.
2541            git::acquire_worktree_config(&repo).await?;
2542            self.state.enabled_worktree_config = true;
2543        }
2544
2545        // `[worktree] setup` hides its products per worktree, which needs the
2546        // same per-worktree config the hook does. Held until fold, like it.
2547        ensure_setup_config(&mut self.state, &repo).await?;
2548
2549        for (index, (spec, label)) in self
2550            .roles
2551            .implementers
2552            .clone()
2553            .into_iter()
2554            .zip(labels)
2555            .enumerate()
2556        {
2557            let branch = self.state.branch_for(label);
2558            let worktree = root.join(format!("cand-{label}"));
2559            // Persist intent before Git can leave a partially created tree.
2560            // Candidate registration follows setup, so it cannot inventory
2561            // crashes or refusals during creation itself.
2562            self.state.event(
2563                "prep-intent",
2564                serde_json::json!({
2565                    "branch": branch,
2566                    "worktree": worktree,
2567                    "start": start,
2568                    "picks": plan.picks,
2569                })
2570                .to_string(),
2571            );
2572            self.state.save()?;
2573            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2574            if self.state.config.blind.commit_msg_hook {
2575                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2576            }
2577            git::local_exclude(&worktree, "/.magi/").await?;
2578            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2579                // Nothing of this prep may survive: a resume must start over,
2580                // not find half the candidates and skip the rest.
2581                git::worktree_remove_clean(&repo, &worktree).await.ok();
2582                git::branch_delete(&repo, &branch).await.ok();
2583                self.rollback_prep(&repo).await;
2584                self.state.status = RunStatus::Blocked;
2585                self.state
2586                    .event("prep", format!("worktree setup failed: {e:#}"));
2587                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2588                self.state.save()?;
2589                return Err(e);
2590            }
2591            self.state.candidates.push(Candidate {
2592                index,
2593                label,
2594                agent: spec.id.clone(),
2595                branch,
2596                worktree,
2597                summary: String::new(),
2598                stat: String::new(),
2599                files: 0,
2600                commits: 0,
2601                empty: false,
2602                failed: None,
2603                verified_noop: None,
2604                duration_ms: 0,
2605                folded: false,
2606            });
2607            self.state.save()?;
2608            for pick in &plan.picks {
2609                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2610                {
2611                    self.state.status = RunStatus::Blocked;
2612                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2613                    self.state
2614                        .event("prep", format!("cannot apply referenced commit: {e}"));
2615                    self.state.save()?;
2616                    return Err(e);
2617                }
2618            }
2619        }
2620
2621        for j in 1..=self.roles.judges.len() {
2622            let wt = root.join(format!("judge-{j}"));
2623            if !wt.exists() {
2624                git::worktree_add_detached(&repo, &wt, &base).await?;
2625                self.setup_seat_worktree(&repo, &wt, true).await?;
2626            }
2627        }
2628
2629        // Disposable, detached checkouts for the design-deliberation stage's
2630        // advisor seats — the same shape as the judges' above, at the same
2631        // base commit, since advisors also only ever read. Sized off the
2632        // configured count directly rather than a resolved roster: unlike
2633        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2634        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2635        // `prep` has no `ResolvedRoles` field to read a count from here.
2636        if self.state.config.graph.advise {
2637            for k in 1..=self.state.config.graph.advisors {
2638                let wt = root.join(format!("advisor-{k}"));
2639                if !wt.exists() {
2640                    git::worktree_add_detached(&repo, &wt, &base).await?;
2641                    self.setup_seat_worktree(&repo, &wt, true).await?;
2642                }
2643            }
2644        }
2645
2646        // A judge cannot tell it is looking at its own patch — the seats keep
2647        // separate conversations — but a panel that shares agents with the
2648        // field is less independent than it looks, and that is worth saying out
2649        // loud once per run rather than leaving it in the config.
2650        let authors: Vec<&str> = self
2651            .roles
2652            .implementers
2653            .iter()
2654            .map(|a| a.id.as_str())
2655            .collect();
2656        let overlap: Vec<String> = self
2657            .roles
2658            .judges
2659            .iter()
2660            .enumerate()
2661            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2662            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2663            .collect();
2664        if !overlap.is_empty() {
2665            let note = format!(
2666                "{} also authored a candidate; blind, but the panel is less \
2667                 independent than {} distinct agents would be",
2668                overlap.join(", "),
2669                self.roles.judges.len()
2670            );
2671            self.state.event("prep", note);
2672        }
2673
2674        self.state.event(
2675            "prep",
2676            format!(
2677                "{} candidates, {} judges, base {} ({})",
2678                self.state.candidates.len(),
2679                self.roles.judges.len(),
2680                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2681                self.state.base_branch
2682            ),
2683        );
2684        self.state.prep_complete = Some(true);
2685        self.state.status = RunStatus::Implementing;
2686        self.state.save()?;
2687        Ok(())
2688    }
2689
2690    // -------------------------------------------------------------- advise
2691
2692    /// The design-deliberation stage: independent, read-only advisor seats
2693    /// each sketch a design before any implementer touches the repository,
2694    /// and (when at least one produced a usable proposal) a synthesis seat
2695    /// blends them into a brief `implement` carries in every candidate's
2696    /// prompt.
2697    ///
2698    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2699    /// advisors` is the proposal count. Everything here is best-effort and
2700    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2701    /// that cannot reach quota, or a synthesis seat that produced nothing
2702    /// usable all leave `implement` exactly as it was before this stage
2703    /// existed — the task instruction alone — rather than failing the whole
2704    /// competition over an enrichment stage. Every outcome is still recorded
2705    /// as an event, so a run that got nothing from this stage says why.
2706    ///
2707    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2708    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2709    /// resumed run whose stage failed would re-run it, and re-spend the
2710    /// agent calls, on every reentry before `implement`.
2711    ///
2712    /// Also skipped once any candidate shows implementation progress — the
2713    /// exact predicate `implement` itself uses to decide a candidate is no
2714    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2715    /// is not enough: a run created by an older binary that predates this
2716    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2717    /// an already-`Implementing`-or-later run under this build would
2718    /// otherwise walk straight back through `prep` (a no-op once candidates
2719    /// exist) into this node and spawn every advisor seat against worktrees
2720    /// `prep` never recreated — after implementation has already started,
2721    /// which is exactly the invariant this stage exists to guarantee.
2722    async fn advise(&mut self) -> Result<()> {
2723        let implement_untouched = self
2724            .state
2725            .candidates
2726            .iter()
2727            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2728        if !self.state.config.graph.advise || self.state.advise_attempted {
2729            return Ok(());
2730        }
2731        if !implement_untouched {
2732            self.state.event(
2733                "advise",
2734                "skipping the design-deliberation stage: at least one \
2735                 candidate already shows implementation progress, so this \
2736                 run is past the point the stage exists to run before"
2737                    .to_owned(),
2738            );
2739            self.state.advise_attempted = true;
2740            self.state.save()?;
2741            return Ok(());
2742        }
2743        let run_id = self.state.id.clone();
2744        let prompts = self.state.config.prompts.clone();
2745        let instruction = self.state.instruction.clone();
2746        let language = self.state.config.graph.language.clone();
2747        let root = self.state.worktree_root();
2748        let n = self.state.config.graph.advisors;
2749        let where_recorded = self.state.dir().join("run.json");
2750
2751        let seats = match self.state.config.advisors() {
2752            Ok(seats) if !seats.is_empty() => seats,
2753            Ok(_) => {
2754                self.state.event(
2755                    "advise",
2756                    format!(
2757                        "[graph] advisors is 0; skipping the design-deliberation \
2758                         stage and continuing without a synthesis brief (see {})",
2759                        where_recorded.display()
2760                    ),
2761                );
2762                self.state.advise_attempted = true;
2763                self.state.save()?;
2764                return Ok(());
2765            }
2766            Err(e) => {
2767                self.state.event(
2768                    "advise",
2769                    format!(
2770                        "could not resolve advisor seats ({e:#}); continuing \
2771                         without a design-deliberation brief (see {})",
2772                        where_recorded.display()
2773                    ),
2774                );
2775                self.state.advise_attempted = true;
2776                self.state.save()?;
2777                return Ok(());
2778            }
2779        };
2780
2781        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2782        let artifacts = agent::artifacts_dir(&self.state.dir());
2783        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2784
2785        let mut jobs = Vec::new();
2786        for (i, spec) in seats.iter().cloned().enumerate() {
2787            let seat_key = format!("advisor-{}", i + 1);
2788            let seat = self.seat(&seat_key, &spec.id);
2789            jobs.push(SeatJob {
2790                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2791                spec,
2792                seat,
2793                cwd: worktrees[i % worktrees.len()].clone(),
2794                timeout,
2795                allow_write: false,
2796                sessions: false,
2797                artifacts: artifacts.clone(),
2798                stem: seat_key,
2799                handover: None,
2800            });
2801        }
2802
2803        self.state.event(
2804            "advise",
2805            format!(
2806                "{} advisor seat(s) sketching a design in parallel",
2807                jobs.len()
2808            ),
2809        );
2810        let mut quota_losses = Vec::new();
2811        let cache = self.state.config.cache_dir();
2812        let ctx = WaveCtx {
2813            carry_seats: false,
2814            pauses: Some((&self.pause, &self.interrupt)),
2815            run: &run_id,
2816            node: "advise",
2817            prompts: &prompts,
2818            cache: cache.as_deref(),
2819            round: None,
2820        };
2821        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2822        let results = ask_json_wave::<Proposal>(
2823            jobs,
2824            Arc::clone(&self.sem),
2825            self.state.config.graph.retries,
2826            &advisor_roster,
2827            &ctx,
2828            &mut quota_losses,
2829            &mut self.state,
2830            &|p: &Proposal| p.validate(),
2831        )
2832        .await;
2833        if self.state.resource_block.is_some() {
2834            return self.stop_for_resource();
2835        }
2836        self.state.quota.extend(quota_losses);
2837
2838        let mut records = Vec::with_capacity(results.len());
2839        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2840            let agent_id = seat.agent.clone();
2841            self.state.seats.insert(seat.key.clone(), seat);
2842            match res {
2843                Ok((proposal, out)) => {
2844                    self.state
2845                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2846                    records.push(advise::AdvisorRecord::proposed(
2847                        i + 1,
2848                        agent_id,
2849                        proposal,
2850                        out.duration_ms,
2851                    ));
2852                }
2853                Err(e) => {
2854                    self.state.event(
2855                        "advise",
2856                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2857                    );
2858                    records.push(advise::AdvisorRecord::failed(
2859                        i + 1,
2860                        agent_id,
2861                        e.to_string(),
2862                    ));
2863                }
2864            }
2865        }
2866
2867        let mut advice = advise::Advice {
2868            records,
2869            synthesis: None,
2870        };
2871        if advice.proposals().is_empty() {
2872            self.state.event(
2873                "advise",
2874                "no advisor produced a usable proposal; continuing without a \
2875                 synthesis brief"
2876                    .to_owned(),
2877            );
2878        } else {
2879            match self
2880                .synthesize_brief(
2881                    &advice,
2882                    &instruction,
2883                    &language,
2884                    &worktrees[0],
2885                    &artifacts,
2886                    &run_id,
2887                    &prompts,
2888                    cache.as_deref(),
2889                )
2890                .await
2891            {
2892                Ok(Some(text)) => {
2893                    self.state.event(
2894                        "advise",
2895                        "synthesized a design brief for the implementer".to_owned(),
2896                    );
2897                    advice.synthesis = Some(text);
2898                }
2899                Ok(None) => {
2900                    self.state.event(
2901                        "advise",
2902                        "the synthesis seat produced nothing usable; continuing \
2903                         without a design brief"
2904                            .to_owned(),
2905                    );
2906                }
2907                Err(e) => {
2908                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2909                        return Err(e);
2910                    }
2911                    self.state.event(
2912                        "advise",
2913                        format!("could not synthesize a design brief: {e:#}"),
2914                    );
2915                }
2916            }
2917        }
2918        advise::apply_reflection(&mut advice);
2919
2920        self.state.advice = Some(advice);
2921        self.state.advise_attempted = true;
2922        self.state.save()?;
2923        Ok(())
2924    }
2925
2926    /// The synthesis seat: reads every advisor's proposal and blends them
2927    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2928    /// out of [`Runner::advise`] only for readability — it is not called
2929    /// anywhere else.
2930    ///
2931    /// Picked the same way [`crate::talk`]'s standing conversation and
2932    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2933    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2934    /// order (a claude seat, else the first runnable agent in roster order)
2935    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2936    /// in [`crate::config`] for why a dedicated field exists here at all.
2937    #[allow(clippy::too_many_arguments)]
2938    async fn synthesize_brief(
2939        &mut self,
2940        advice: &advise::Advice,
2941        instruction: &str,
2942        language: &str,
2943        cwd: &Path,
2944        artifacts: &Path,
2945        run_id: &str,
2946        prompts: &Prompts,
2947        cache: Option<&Path>,
2948    ) -> Result<Option<String>> {
2949        let chain = agent::pick_chain(
2950            &self.state.config.agents,
2951            self.state.config.roles.synthesizer.as_ref(),
2952            &agent::installed,
2953            "synthesizer",
2954        )?;
2955        let proposals = advice.proposals();
2956        let mut prompt = prompt::with_overlay(
2957            prompt::synthesize_brief(instruction, &proposals, language),
2958            prompts.overlay("advise"),
2959        );
2960        if cache.is_some() {
2961            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2962            // below — see `prompt::build_cache_note`'s doc for why telling a
2963            // read-only seat to build through the shared cache is exactly how
2964            // a sandbox's write refusal gets misread as a defect.
2965            prompt.push('\n');
2966            prompt.push_str(&prompt::build_cache_note("advise", false));
2967        }
2968        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2969        // Each id is tried once, in order; a quota hit, error or unusable
2970        // answer moves to the next. The seat is single-turn (`sessions:
2971        // false`) and the prompt is the whole context, so a fallback agent
2972        // needs nothing carried over.
2973        let mut last = None;
2974        for (n, spec) in chain.iter().enumerate() {
2975            if n > 0 {
2976                self.state
2977                    .event("advise", format!("synthesis falling back to {}", spec.id));
2978            }
2979            let mut seat = self.seat("advise-synthesis", &spec.id);
2980            let wait = self.repository_wait();
2981            let outcome = wait
2982                .scope(agent::invoke(
2983                    spec,
2984                    &mut seat,
2985                    &Invocation {
2986                        coordination: self.state.config.coordination.as_ref(),
2987                        fetched_base: None,
2988                        cwd,
2989                        prompt: &prompt,
2990                        timeout,
2991                        allow_write: false,
2992                        sessions: false,
2993                        artifacts,
2994                        stem: &if n == 0 {
2995                            "advise-synthesis".to_owned()
2996                        } else {
2997                            format!("advise-synthesis-{}", spec.id)
2998                        },
2999                        run: run_id,
3000                        node: "advise",
3001                        cache_dir: None,
3002                        attachments: &[],
3003                        writable: &[],
3004                    },
3005                ))
3006                .await;
3007            if outcome.is_ok() {
3008                self.state.seats.insert(seat.key.clone(), seat);
3009            }
3010            let advance = agent::chain_advances(&outcome);
3011            last = Some(outcome);
3012            if !advance {
3013                break;
3014            }
3015        }
3016        // Exhausted: the last attempt's result is what a single failed seat
3017        // would have produced.
3018        let out = last.expect("a chain holds at least one agent")?;
3019        if !out.usable() {
3020            return Ok(None);
3021        }
3022        let text =
3023            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3024        Ok((!text.trim().is_empty()).then_some(text))
3025    }
3026
3027    // ----------------------------------------------------------- implement
3028
3029    async fn implement(&mut self) -> Result<()> {
3030        self.persist_candidates().await?;
3031        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3032        // agent files with `magi task add` name the run that paid for it. The
3033        // prompt overlay is cloned alongside it because the waves borrow it
3034        // while `self` is mutably borrowed by the node's own bookkeeping.
3035        let run_id = self.state.id.clone();
3036        let prompts = self.state.config.prompts.clone();
3037        let todo: Vec<usize> = self
3038            .state
3039            .candidates
3040            .iter()
3041            .enumerate()
3042            .filter(|(_, c)| {
3043                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3044                    && c.failed.is_none()
3045                    && !c.empty
3046            })
3047            .map(|(i, _)| i)
3048            .collect();
3049        if todo.is_empty() {
3050            return self.after_implement();
3051        }
3052        self.state.status = RunStatus::Implementing;
3053
3054        let language = self.state.config.graph.language.clone();
3055        let github_language = self.state.config.graph.github_language.clone();
3056        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3057        let sessions = self.state.config.graph.sessions;
3058        let artifacts = agent::artifacts_dir(&self.state.dir());
3059        // The design-deliberation stage's blended brief, when `advise` found
3060        // one — carried into every implementer's prompt the same way
3061        // regardless of which candidate it is.
3062        let brief = self
3063            .state
3064            .advice
3065            .as_ref()
3066            .and_then(|a| a.synthesis.as_deref())
3067            .map(str::to_owned);
3068        let attachments = self.state.attachments.clone();
3069
3070        let mut jobs = Vec::new();
3071        for &i in &todo {
3072            let (index, label, worktree) = {
3073                let c = &self.state.candidates[i];
3074                (c.index, c.label, c.worktree.clone())
3075            };
3076            let spec = self.roles.implementers[index].clone();
3077            let seat_key = format!("impl-{label}");
3078            let seat = self.seat(&seat_key, &spec.id);
3079            let instruction = seeded_instruction(&self.state);
3080            jobs.push(SeatJob {
3081                spec,
3082                seat,
3083                prompt: prompt::implement_in(
3084                    &instruction,
3085                    &worktree.to_string_lossy(),
3086                    &language,
3087                    &github_language,
3088                    brief.as_deref(),
3089                    &attachments,
3090                ),
3091                cwd: worktree,
3092                timeout,
3093                allow_write: true,
3094                sessions,
3095                artifacts: artifacts.clone(),
3096                stem: format!("impl-{label}"),
3097                handover: None,
3098            });
3099        }
3100
3101        self.state.event(
3102            "implement",
3103            format!("{} candidates in parallel", jobs.len()),
3104        );
3105        // Kept so a seat whose CLI hung up can be asked again from the same
3106        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
3107        // can update a seat's own entry once a fallback agent takes it over —
3108        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
3109        // whichever agent actually answered, not the one that quota'd out.
3110        let mut sent = jobs.clone();
3111        let cache = self.state.config.cache_dir();
3112        let ctx = WaveCtx {
3113            carry_seats: false,
3114            pauses: Some((&self.pause, &self.interrupt)),
3115            run: &run_id,
3116            node: "implement",
3117            prompts: &prompts,
3118            cache: cache.as_deref(),
3119            round: None,
3120        };
3121        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3122        if !wave_blocked(&results) {
3123            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3124                .await;
3125            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3126                .await?;
3127            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3128                .await;
3129        }
3130        // Do not finalize failed candidates when the entire wave needs machine
3131        // recovery: leaving them pending resumes this node with its work intact.
3132        if !results.is_empty()
3133            && results
3134                .iter()
3135                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3136        {
3137            let losses: Vec<_> = results
3138                .iter()
3139                .map(|(_, seat, out)| {
3140                    let AgentOutcome::Quota(o) = out else {
3141                        unreachable!()
3142                    };
3143                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3144                })
3145                .collect();
3146            self.state.quota.extend(losses.clone());
3147            for (_, seat, _) in results {
3148                self.state.seats.insert(seat.key.clone(), seat);
3149            }
3150            for &i in &todo {
3151                let label = self.state.candidates[i].label;
3152                self.state.interrupted_implementations.insert(label);
3153                self.state
3154                    .candidate_preservation
3155                    .insert(label, "machine outage awaiting persistence".into());
3156            }
3157            // Preserve partial edits too, while keeping every interrupted seat
3158            // pending even when its tree already contains an inherited patch.
3159            block_machine_outage(&mut self.state, "implement", &losses);
3160            self.state.save()?;
3161            self.persist_candidates().await?;
3162            for &i in &todo {
3163                self.state
3164                    .interrupted_implementations
3165                    .insert(self.state.candidates[i].label);
3166            }
3167            return self.stop_for_resource();
3168        }
3169        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3170
3171        let selected_failed = self.state.config.graph.selected_implementer.is_some()
3172            && results
3173                .iter()
3174                .any(|(_, _, out)| FailClass::of(out).is_some());
3175        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3176            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3177                // Executed turns interrupted by disk pressure already carry
3178                // a preservation barrier from wave. Admission-only waits do
3179                // not. Keep the interrupted seat's session for recovery too.
3180                if self
3181                    .state
3182                    .candidate_preservation
3183                    .contains_key(&self.state.candidates[i].label)
3184                {
3185                    self.state.seats.insert(seat.key.clone(), seat);
3186                }
3187                continue;
3188            }
3189            let seat_key = seat.key.clone();
3190            // A quota fallback (`resume_seat_handovers`) may have handed this
3191            // seat to a different agent than the one `prep` recorded on the
3192            // candidate; the stats tables and any later fixer-defaults-to-
3193            // winner's-author lookup must credit whoever actually answered —
3194            // unless every fallback also quota'd out, in which case nobody
3195            // actually answered and crediting the last agent tried would
3196            // erase every earlier agent's own quota loss from the stats
3197            // tables instead of just this one seat's.
3198            let agent = seat.agent.clone();
3199            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3200            self.state.seats.insert(seat.key.clone(), seat);
3201            let label = self.state.candidates[i].label;
3202            self.state.interrupted_implementations.remove(&label);
3203
3204            let (summary, duration, failed, verified_claim) = match out {
3205                AgentOutcome::Ok(o) => {
3206                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3207                    let failed = (!o.usable()).then(|| {
3208                        if o.timed_out {
3209                            "agent timed out".to_owned()
3210                        } else {
3211                            format!(
3212                                "agent exited with {}",
3213                                crate::notices::exit_status(o.exit_code)
3214                            )
3215                        }
3216                    });
3217                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3218                    (text, o.duration_ms, failed, verified_claim)
3219                }
3220                // Left un-resumed by `resume_undelivered` (a dirty tree
3221                // already rescues the work, or there was no session left to
3222                // resume into) — reported like the ordinary failure it is,
3223                // never as if `o.text` (the CLI's raw error JSON) were an
3224                // answer.
3225                AgentOutcome::Dropped(o) => {
3226                    let why = o
3227                        .dropped
3228                        .as_ref()
3229                        .map(|d| d.why.as_str())
3230                        .unwrap_or("the CLI ended the stream without delivering its answer");
3231                    (
3232                        String::new(),
3233                        o.duration_ms,
3234                        Some(format!("the CLI dropped the stream ({why})")),
3235                        None,
3236                    )
3237                }
3238                AgentOutcome::Quota(o) => {
3239                    self.state.quota.push(QuotaLoss::from_report(
3240                        seat_key,
3241                        "implement",
3242                        o.quota.as_ref(),
3243                    ));
3244                    (
3245                        String::new(),
3246                        o.duration_ms,
3247                        Some("rate limited (quota); produced no change".to_owned()),
3248                        None,
3249                    )
3250                }
3251                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3252                    (String::new(), 0, Some(e), None)
3253                }
3254            };
3255
3256            // Save every completed turn before any fallible Git operation. A failed
3257            // rescue must not erase the reply or prevent collection of other seats.
3258            let c = &mut self.state.candidates[i];
3259            if !exhausted_the_fallback_chain {
3260                c.agent = agent;
3261            }
3262            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3263            c.duration_ms = duration;
3264            c.failed = failed;
3265            c.verified_noop = verified_claim;
3266            self.state
3267                .candidate_preservation
3268                .insert(label, "awaiting persistence".to_owned());
3269        }
3270        self.state.save()?;
3271        self.persist_candidates().await?;
3272
3273        if resource_blocked {
3274            return self.stop_for_resource();
3275        }
3276        if selected_failed {
3277            self.state.status = RunStatus::Blocked;
3278            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3279            self.state.save()?;
3280            bail!(
3281                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3282            );
3283        }
3284        self.after_implement()
3285    }
3286
3287    /// Retry only persistence after a completed implementation turn.
3288    async fn persist_candidates(&mut self) -> Result<()> {
3289        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3290        for label in labels {
3291            let i = self
3292                .state
3293                .candidates
3294                .iter()
3295                .position(|c| c.label == label)
3296                .context("pending candidate is missing")?;
3297            let c = self.state.candidates[i].clone();
3298            let interrupted = self.state.interrupted_implementations.contains(&label);
3299            let result = async {
3300                let wait = crate::repo_lock::WaitPolicy::new(
3301                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3302                    || false,
3303                );
3304                let _coordination =
3305                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3306                // Best effort evidence requires no index writes. A successful snapshot
3307                // never authorizes removal of a tree whose rescue failed.
3308                if let Err(e) = git::snapshot_uncommitted(
3309                    &c.worktree,
3310                    &self
3311                        .state
3312                        .dir()
3313                        .join("artifacts")
3314                        .join(format!("candidate-{label}-recovery")),
3315                )
3316                .await
3317                {
3318                    self.state.event(
3319                        "preserve",
3320                        format!("candidate {label} snapshot incomplete: {e:#}"),
3321                    );
3322                }
3323                let rescued = git::rescue_commit(
3324                    &c.worktree,
3325                    &format!("magi: candidate {label} (uncommitted work)"),
3326                )
3327                .await?;
3328                self.state.note_withheld("implement", &rescued.withheld);
3329                let base = &self.state.base_commit;
3330                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3331                let patch = git::diff(&c.worktree, base, "HEAD").await?;
3332                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3333                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3334                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3335                if c.empty && commits > 0 && !patch.trim().is_empty()
3336                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
3337                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3338                        || self.state.tally.is_some())
3339                {
3340                    // Keep the old classification and pending marker so every
3341                    // retry stops too. Retain decisions as recovery evidence;
3342                    // only a supported recovery operation may replace them.
3343                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3344                }
3345                let c = &mut self.state.candidates[i];
3346                c.commits = commits;
3347                c.stat = stat;
3348                c.files = files;
3349                let no_changes = commits == 0 || patch.trim().is_empty();
3350                c.empty = no_changes && !interrupted;
3351                if interrupted && no_changes {
3352                    c.failed = None;
3353                    c.verified_noop = None;
3354                }
3355                if !c.empty {
3356                    c.failed = None;
3357                    c.verified_noop = None;
3358                }
3359                self.state.event(
3360                    "preserve",
3361                    format!("candidate {label}: {files} files, {commits} commits"),
3362                );
3363                Ok::<_, anyhow::Error>(())
3364            }
3365            .await;
3366            match result {
3367                Ok(()) => {
3368                    self.state.candidate_preservation.remove(&label);
3369                    if !interrupted || self.state.candidates[i].files > 0 {
3370                        self.state.interrupted_implementations.remove(&label);
3371                    }
3372                }
3373                Err(e) => {
3374                    let reason = format!("candidate {label} preservation failed: {e:#}");
3375                    self.state.event("preserve", &reason);
3376                    self.state.candidate_preservation.insert(label, reason);
3377                }
3378            }
3379            if self.state.candidate_preservation.is_empty()
3380                && self.state.status == RunStatus::Blocked
3381                && self.state.candidates.iter().all(|c| c.failed.is_none())
3382            {
3383                // Persist the recovered phase with the cleared pending marker;
3384                // a crash between saves must not turn this into a selected-seat
3385                // failure hold on the next resume.
3386                self.state.status = RunStatus::Implementing;
3387                self.state.event(
3388                    "preserve",
3389                    "candidate persistence recovered; implementation can continue",
3390                );
3391            }
3392            self.state.save()?;
3393        }
3394        if !self.state.candidate_preservation.is_empty() {
3395            self.state.status = RunStatus::Blocked;
3396            self.state.save()?;
3397            bail!(
3398                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3399            );
3400        }
3401        Ok(())
3402    }
3403
3404    /// Ask again, once, for work a CLI did and then failed to hand over.
3405    ///
3406    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3407    /// status with an empty response and a usage report showing output tokens,
3408    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3409    /// seven minutes and 14,267 output tokens that arrived as an empty
3410    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3411    ///
3412    /// Two conditions, and both matter:
3413    ///
3414    /// - **Only when the tree is untouched.** Often the agent has already
3415    ///   written its files and only the closing message was lost; the rescue
3416    ///   commit below picks that up and there is nothing to ask for. Re-asking
3417    ///   then would pay for a second implementation of work already on disk.
3418    /// - **Once.** A CLI that drops one stream can drop the next, and this
3419    ///   node is the most expensive in the graph.
3420    ///
3421    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3422    /// dropped reply still carried its `conversation_id`, so the seat is asked
3423    /// to finish what it was doing rather than sent the whole task again. It
3424    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3425    /// node's - for the same reason a re-ranked judge does: restating finished
3426    /// work is not the work.
3427    ///
3428    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3429    /// same way until it resets, while an abandoned conversation is still
3430    /// there to be picked up.
3431    async fn resume_undelivered(
3432        &mut self,
3433        results: &mut [(usize, SeatState, AgentOutcome)],
3434        sent: &[SeatJob],
3435        prompts: &Prompts,
3436        run_id: &str,
3437    ) {
3438        for (wi, seat, out) in results.iter_mut() {
3439            let Some(dropped) = (match &*out {
3440                AgentOutcome::Dropped(o) => o.dropped.clone(),
3441                _ => None,
3442            }) else {
3443                continue;
3444            };
3445            let Some(job) = sent.get(*wi) else { continue };
3446            // Already on disk? Then only the closing message was lost.
3447            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3448                self.state.event(
3449                    "implement",
3450                    format!(
3451                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3452                         work is in the tree",
3453                        seat.key, dropped.output_tokens, dropped.why
3454                    ),
3455                );
3456                continue;
3457            }
3458            // The re-ask only makes sense as a resume: `resume_after_drop`
3459            // says nothing about the task, trusting the seat to still hold it.
3460            // Without a session to resume — sessions disabled, or this CLI's
3461            // drop shape happened not to carry a session id — that prompt
3462            // would open a brand-new conversation with no context at all,
3463            // which is worse than leaving this as the ordinary failure it
3464            // already is.
3465            if !has_context(&job.spec, seat, job.sessions) {
3466                self.state.event(
3467                    "implement",
3468                    format!(
3469                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3470                         is no session left to resume",
3471                        seat.key, dropped.output_tokens, dropped.why
3472                    ),
3473                );
3474                continue;
3475            }
3476            self.state.event(
3477                "implement",
3478                format!(
3479                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3480                     conversation",
3481                    seat.key, dropped.output_tokens, dropped.why
3482                ),
3483            );
3484            let mut retry = job.clone();
3485            retry.seat = seat.clone();
3486            retry.prompt = prompt::resume_after_drop(&dropped.why);
3487            retry.timeout = retry_budget(job.timeout, true);
3488            retry.stem = format!("{}-resume", job.stem);
3489            let cache = self.state.config.cache_dir();
3490            let ctx = WaveCtx {
3491                carry_seats: false,
3492                pauses: Some((&self.pause, &self.interrupt)),
3493                run: run_id,
3494                node: "implement",
3495                prompts,
3496                cache: cache.as_deref(),
3497                round: None,
3498            };
3499            let (resumed_seat, resumed) =
3500                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3501            *seat = resumed_seat;
3502            *out = resumed;
3503        }
3504    }
3505
3506    /// Fall an implement seat through to the next untried agent in the
3507    /// implementer roster when it lost to quota — or, since the handover was
3508    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3509    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3510    /// quota path itself being unchanged — instead of leaving the
3511    /// seat's loss final the moment one agent's account runs dry.
3512    ///
3513    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3514    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3515    /// truncates to the single slot rotation picked, so a solo task whose one
3516    /// implementer hits quota mid-run used to have nothing else to try. This
3517    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3518    /// unrotated roster — which is the only place the *other* candidates in
3519    /// the machine's roster still exist once `implementers` has been cut down
3520    /// to size.
3521    ///
3522    /// Walks forward from just past the seat's own original position in the
3523    /// roster, never wrapping back to the front: a later candidate slot (say
3524    /// `beta`, the roster's second entry) must fall through to the *next*
3525    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3526    /// almost certainly a different candidate's own agent already — and once
3527    /// the roster's tail is exhausted there is nothing left to fall through
3528    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3529    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3530    /// let this retry that id forever. The loop keeps falling through until
3531    /// an attempt lands something other than `Quota` or the roster's tail
3532    /// runs out of untried ids, at which point the seat is left exactly as
3533    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3534    /// `QuotaLoss` recorded, the candidate failed/empty.
3535    ///
3536    /// `sent` is taken mutably and updated with the fallback agent's spec:
3537    /// `resume_unconfirmed_commands`, which runs after this and also reads
3538    /// `sent`, must see whichever agent actually ended up answering the seat
3539    /// — reading the stale, original spec there would check session
3540    /// eligibility against the wrong CLI and could hand a fallback agent's
3541    /// session id to the agent that just lost the seat to quota.
3542    ///
3543    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3544    /// — `self.seat` only reuses state when the agent id is unchanged, so
3545    /// handing it a different id already gets this for free. Reusing the old
3546    /// seat would resume a different CLI's session as if it were a
3547    /// continuation of this one.
3548    ///
3549    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3550    /// a quota loss cuts an agent off mid-turn, so anything already in the
3551    /// tree is unfinished work, not a completed candidate a re-ask would pay
3552    /// for twice. A dirty tree is rescued into a commit first (the same
3553    /// neutral-identity rescue `implement`'s own outcome loop gives every
3554    /// candidate) so the next agent starts clean.
3555    ///
3556    /// The new agent gets the implementer's full prompt and full
3557    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3558    /// it has no session and no context, and is implementing the task from
3559    /// nothing, unlike a resumed drop which is only restating work already
3560    /// done.
3561    ///
3562    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3563    /// `implement` event, never into `self.state.quota` — that is what
3564    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3565    /// go unspent, and a seat that ultimately recovered on its second or
3566    /// third agent is not the stalled panel that check exists to catch. Only
3567    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3568    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3569    /// outcome loop already has — this helper never pushes to it itself.
3570    async fn resume_seat_handovers(
3571        &mut self,
3572        results: &mut [(usize, SeatState, AgentOutcome)],
3573        sent: &mut [SeatJob],
3574        prompts: &Prompts,
3575        run_id: &str,
3576    ) -> Result<()> {
3577        let instruction = seeded_instruction(&self.state);
3578        let language = self.state.config.graph.language.clone();
3579        let github_language = self.state.config.graph.github_language.clone();
3580        let brief = self
3581            .state
3582            .advice
3583            .as_ref()
3584            .and_then(|a| a.synthesis.as_deref())
3585            .map(str::to_owned);
3586        let attachments = self.state.attachments.clone();
3587        for (wi, seat, out) in results.iter_mut() {
3588            // Who holds the other candidate seats of this wave right now
3589            // (earlier handovers already written back to `sent`).
3590            let others: BTreeSet<String> = sent
3591                .iter()
3592                .enumerate()
3593                .filter(|(j, _)| j != wi)
3594                .map(|(_, j)| j.spec.id.clone())
3595                .collect();
3596            let Some(job) = sent.get_mut(*wi) else {
3597                continue;
3598            };
3599            // Where the seat's own original agent sits in the roster — the
3600            // fallback walk starts just past here, never at the front, so a
3601            // later candidate slot's quota loss does not fall back onto an
3602            // earlier slot's own agent.
3603            let start = self
3604                .roles
3605                .implementer_roster
3606                .iter()
3607                .position(|s| s.id == job.spec.id)
3608                .unwrap_or(0);
3609            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3610            let mut fallback_attempt = 0usize;
3611            let mut prev: Option<FailClass> = None;
3612            while let Some(cur) = FailClass::of(&*out) {
3613                if !should_hand_over(prev.as_ref(), &cur) {
3614                    break;
3615                }
3616                let Some(next) =
3617                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3618                        .cloned()
3619                else {
3620                    break;
3621                };
3622                tried.insert(next.id.clone());
3623                fallback_attempt += 1;
3624
3625                let coordination = self.coordinate(&job.cwd).await?;
3626                if let Ok(r) = git::rescue_commit(
3627                    &job.cwd,
3628                    &format!(
3629                        "magi: candidate {} (uncommitted work before {} fallback)",
3630                        seat.key,
3631                        if cur == FailClass::Quota {
3632                            "quota"
3633                        } else {
3634                            "handover"
3635                        }
3636                    ),
3637                )
3638                .await
3639                {
3640                    self.state.note_withheld("implement", &r.withheld);
3641                }
3642                drop(coordination);
3643
3644                record_handover(
3645                    &mut self.state,
3646                    "implement",
3647                    &seat.key,
3648                    &seat.agent,
3649                    &next.id,
3650                    &cur,
3651                    &fail_reason(&*out),
3652                );
3653                prev = Some(cur.clone());
3654
3655                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3656                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3657                // Kept in sync on `sent` itself, not just the local retry: a
3658                // later helper (`resume_unconfirmed_commands`) reads `sent`
3659                // after this one returns and must see whichever agent is now
3660                // occupying the seat, not the one that just quota'd out —
3661                // otherwise it would judge session/continuation eligibility
3662                // by the wrong CLI and could resend a fallback's session id
3663                // to the agent that lost it the seat in the first place.
3664                job.spec = next.clone();
3665                let mut retry = job.clone();
3666                retry.seat = new_seat;
3667                retry.prompt = prompt::implement_in(
3668                    &instruction,
3669                    &job.cwd.to_string_lossy(),
3670                    &language,
3671                    &github_language,
3672                    brief.as_deref(),
3673                    &attachments,
3674                );
3675                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3676                let cache = self.state.config.cache_dir();
3677                let ctx = WaveCtx {
3678                    carry_seats: false,
3679                    pauses: Some((&self.pause, &self.interrupt)),
3680                    run: run_id,
3681                    node: "implement",
3682                    prompts,
3683                    cache: cache.as_deref(),
3684                    round: None,
3685                };
3686                let (fallback_seat, fallback_out) = run_one(
3687                    retry,
3688                    Arc::clone(&self.sem),
3689                    &ctx,
3690                    &mut self.state,
3691                    fallback_attempt,
3692                )
3693                .await;
3694                *seat = fallback_seat;
3695                *out = fallback_out;
3696            }
3697        }
3698        Ok(())
3699    }
3700
3701    /// Ask an implement seat's own CLI to confirm what it started, once, when
3702    /// its reply reported a command whose completion status it never
3703    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3704    /// that does and does not mean.
3705    ///
3706    /// The completion contract this task asks for, extended to `implement`
3707    /// with the same signal `continue_fix_report` reads for the fixer,
3708    /// rather than a keyword search over the reply or a hard requirement on
3709    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3710    /// a candidate's CLI turn ended cleanly while a test run it had started
3711    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3712    /// named in it at all is untouched by this: `commands` is empty, so
3713    /// there is nothing to be unconfirmed.
3714    ///
3715    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3716    /// this is not about recovering edits that might already be on disk, it
3717    /// is about a result the seat itself never vouched for, which resuming
3718    /// asks for regardless of what the tree already holds. Bounded to one
3719    /// attempt for the same reason `resume_undelivered` is — this is the
3720    /// most expensive node in the graph — and a seat that still cannot
3721    /// confirm on that attempt is left as whatever its (possibly still
3722    /// unconfirmed) reply says; this does not invent a new "failed" reason
3723    /// for a candidate that otherwise produced a real, committed change.
3724    async fn resume_unconfirmed_commands(
3725        &mut self,
3726        results: &mut [(usize, SeatState, AgentOutcome)],
3727        sent: &[SeatJob],
3728        prompts: &Prompts,
3729        run_id: &str,
3730    ) {
3731        for (wi, seat, out) in results.iter_mut() {
3732            let AgentOutcome::Ok(o) = &*out else {
3733                continue;
3734            };
3735            if !has_unconfirmed_command(&o.commands) {
3736                continue;
3737            }
3738            let Some(job) = sent.get(*wi) else { continue };
3739            if !has_context(&job.spec, seat, job.sessions) {
3740                self.state.event(
3741                    "implement",
3742                    format!(
3743                        "{}: the reply named a command whose own CLI never confirmed the exit \
3744                         status of, but there is no session left to resume",
3745                        seat.key
3746                    ),
3747                );
3748                continue;
3749            }
3750            self.state.event(
3751                "implement",
3752                format!(
3753                    "{}: the reply named a command whose own CLI never confirmed the exit \
3754                     status of; resuming the conversation",
3755                    seat.key
3756                ),
3757            );
3758            let mut retry = job.clone();
3759            retry.seat = seat.clone();
3760            retry.prompt = prompt::resume_incomplete(
3761                "a command in your last reply had no confirmed exit status",
3762            );
3763            retry.timeout = retry_budget(job.timeout, true);
3764            retry.stem = format!("{}-confirm", job.stem);
3765            let cache = self.state.config.cache_dir();
3766            let ctx = WaveCtx {
3767                carry_seats: false,
3768                pauses: Some((&self.pause, &self.interrupt)),
3769                run: run_id,
3770                node: "implement",
3771                prompts,
3772                cache: cache.as_deref(),
3773                round: None,
3774            };
3775            let (resumed_seat, resumed) =
3776                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3777            *seat = resumed_seat;
3778            *out = resumed;
3779        }
3780    }
3781
3782    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3783    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3784    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3785    /// that motivated this.
3786    ///
3787    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3788    /// own nudge loop already covers for judge/review/vote seats, and not a
3789    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3790    /// implement seats: here the CLI turn genuinely finished while the node's
3791    /// own work — the fixer's account of what it did — had not. Gated purely
3792    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3793    /// reply, never on any wording in it, so a fixer whose valid, first-try
3794    /// `FixReport` happens to mention having waited on a background test is
3795    /// never resumed — the `Ok(report)` branch at the call site returns
3796    /// before this is ever invoked.
3797    ///
3798    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3799    /// attempt ([`retry_budget`]), nothing attempted once the session is
3800    /// gone, and a quota hit ends the loop immediately rather than retrying a
3801    /// rate limit that fails the same way again.
3802    async fn continue_fix_report(
3803        &mut self,
3804        mut seat: SeatState,
3805        parse_err: String,
3806        job: &SeatJob,
3807        prompts: &Prompts,
3808        run_id: &str,
3809        round: usize,
3810    ) -> (
3811        SeatState,
3812        Option<FixReport>,
3813        Option<String>,
3814        ContinuationRecord,
3815    ) {
3816        let mut last_err = parse_err;
3817        let mut cumulative_wait_ms = 0u64;
3818        let mut attempts = 0usize;
3819        loop {
3820            if !has_context(&job.spec, &seat, job.sessions) {
3821                self.state.event(
3822                    "fix",
3823                    format!(
3824                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3825                         session left to resume into"
3826                    ),
3827                );
3828                let outcome = if attempts == 0 {
3829                    ContinuationOutcome::NoSession
3830                } else {
3831                    ContinuationOutcome::Exhausted
3832                };
3833                return (
3834                    seat,
3835                    None,
3836                    Some(format!("unparsable fix report: {last_err}")),
3837                    ContinuationRecord {
3838                        attempts,
3839                        cumulative_wait_ms,
3840                        outcome,
3841                    },
3842                );
3843            }
3844            if attempts >= MAX_FIX_CONTINUATIONS {
3845                self.state.event(
3846                    "fix",
3847                    format!(
3848                        "round {round}: fixer's reply still had no adoption report after \
3849                         {attempts} continuation(s) ({last_err}); giving up"
3850                    ),
3851                );
3852                return (
3853                    seat,
3854                    None,
3855                    Some(format!(
3856                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3857                    )),
3858                    ContinuationRecord {
3859                        attempts,
3860                        cumulative_wait_ms,
3861                        outcome: ContinuationOutcome::Exhausted,
3862                    },
3863                );
3864            }
3865            attempts += 1;
3866            self.state.event(
3867                "fix",
3868                format!(
3869                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3870                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3871                ),
3872            );
3873            let mut retry = job.clone();
3874            retry.seat = seat.clone();
3875            retry.prompt = prompt::resume_incomplete(&last_err);
3876            retry.timeout = retry_budget(job.timeout, true);
3877            retry.stem = format!("{}-continue{attempts}", job.stem);
3878            let cache = self.state.config.cache_dir();
3879            let ctx = WaveCtx {
3880                carry_seats: false,
3881                pauses: Some((&self.pause, &self.interrupt)),
3882                run: run_id,
3883                node: "fix",
3884                prompts,
3885                cache: cache.as_deref(),
3886                round: Some(round),
3887            };
3888            let (resumed_seat, resumed_out) = run_one(
3889                retry,
3890                Arc::clone(&self.sem),
3891                &ctx,
3892                &mut self.state,
3893                attempts,
3894            )
3895            .await;
3896            seat = resumed_seat;
3897            match resumed_out {
3898                AgentOutcome::Ok(o) => {
3899                    cumulative_wait_ms += o.duration_ms;
3900                    match verdict::extract_json::<FixReport>(&o.text) {
3901                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3902                            self.state.event(
3903                                "fix",
3904                                format!(
3905                                    "round {round}: fixer's adoption report recovered after \
3906                                     {attempts} continuation(s)"
3907                                ),
3908                            );
3909                            return (
3910                                seat,
3911                                Some(report),
3912                                None,
3913                                ContinuationRecord {
3914                                    attempts,
3915                                    cumulative_wait_ms,
3916                                    outcome: ContinuationOutcome::Resumed,
3917                                },
3918                            );
3919                        }
3920                        // The report parsed, but this same reply's own
3921                        // CommandEvidence — the identical record `state.jobs`
3922                        // renders — names a command whose CLI never
3923                        // confirmed an exit status. Read together, that is
3924                        // not a resolved answer: keep nudging rather than
3925                        // accept a report standing next to a command the
3926                        // seat's own CLI cannot vouch for.
3927                        Ok(_) => {
3928                            last_err = "the reply parsed, but it reported a command whose own CLI \
3929                                 never confirmed an exit status"
3930                                .to_owned();
3931                        }
3932                        Err(e) => last_err = e.to_string(),
3933                    }
3934                }
3935                AgentOutcome::Quota(o) => {
3936                    cumulative_wait_ms += o.duration_ms;
3937                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3938                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3939                    self.state.quota.push(loss);
3940                    self.state.event(
3941                        "fix",
3942                        format!(
3943                            "round {round}: continuation rate limited (quota); not retrying now"
3944                        ),
3945                    );
3946                    return (
3947                        seat,
3948                        None,
3949                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3950                        ContinuationRecord {
3951                            attempts,
3952                            cumulative_wait_ms,
3953                            outcome: ContinuationOutcome::QuotaLost,
3954                        },
3955                    );
3956                }
3957                AgentOutcome::Dropped(o) => {
3958                    cumulative_wait_ms += o.duration_ms;
3959                    let why = o
3960                        .dropped
3961                        .as_ref()
3962                        .map(|d| d.why.as_str())
3963                        .unwrap_or("the CLI ended the stream without delivering its answer");
3964                    last_err = format!("the CLI dropped the stream ({why})");
3965                }
3966                AgentOutcome::Failed(e) => last_err = e,
3967                // The continuation was never started; the report stays lost
3968                // and the caller records that, without any handover.
3969                AgentOutcome::ResourceBlocked(e) => {
3970                    return (
3971                        seat,
3972                        None,
3973                        Some(format!("the continuation was not started: {e}")),
3974                        ContinuationRecord {
3975                            attempts: attempts.saturating_sub(1),
3976                            cumulative_wait_ms,
3977                            outcome: ContinuationOutcome::NoSession,
3978                        },
3979                    );
3980                }
3981            }
3982        }
3983    }
3984
3985    fn after_implement(&mut self) -> Result<()> {
3986        if !self.state.candidate_preservation.is_empty() {
3987            bail!("candidate preservation incomplete; judging is unsafe");
3988        }
3989        // Scan every candidate patch once the set is complete.
3990        if self.state.leaks.is_empty() {
3991            let cfg = self.state.config.blind.clone();
3992            let mut leaks = Vec::new();
3993            for c in &self.state.candidates {
3994                let Some(patch) =
3995                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3996                else {
3997                    continue;
3998                };
3999                leaks.extend(blind::scan(
4000                    &format!("candidate {} patch", c.label),
4001                    &patch,
4002                    &cfg.vendor_tokens,
4003                ));
4004            }
4005            if !leaks.is_empty() {
4006                let summary = leaks
4007                    .iter()
4008                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4009                    .collect::<Vec<_>>()
4010                    .join(", ");
4011                match cfg.on_leak {
4012                    LeakPolicy::Fail => {
4013                        self.state.status = RunStatus::Failed;
4014                        self.state
4015                            .event("blind", format!("vendor text in a patch: {summary}"));
4016                        self.state.leaks = leaks;
4017                        self.state.save()?;
4018                        self.settle_questions();
4019                        bail!(
4020                            "blind.on_leak = \"fail\" and vendor text reached a \
4021                             judged patch: {summary}"
4022                        );
4023                    }
4024                    LeakPolicy::Redact => self.state.event(
4025                        "blind",
4026                        format!("redacting vendor text for judging: {summary}"),
4027                    ),
4028                    LeakPolicy::Warn => self.state.event(
4029                        "blind",
4030                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
4031                    ),
4032                }
4033                self.state.leaks = leaks;
4034            }
4035        }
4036
4037        if self.state.viable().is_empty() {
4038            if self.state.all_candidates_verified_noop() {
4039                // Every candidate agreed, with evidence the adoption guard
4040                // accepted, that nothing belongs in this worktree. That is
4041                // not the same fact as a candidate that simply failed to
4042                // write anything, and settling it as an ordinary `Failed`
4043                // (see `SCHEMA`'s doc for schema 10) is what let two of
4044                // task 391f's attempts burn a retry each re-discovering the
4045                // same already-landed fix. Terminal either way, so `judge`
4046                // must never run over an empty candidate set — unlike the
4047                // `Failed` branch below this returns `Ok`, not an error:
4048                // nothing here failed.
4049                self.state.status = RunStatus::VerifiedNoop;
4050                self.state.save()?;
4051                self.settle_questions();
4052                return Ok(());
4053            }
4054            self.state.status = RunStatus::Failed;
4055            self.state.save()?;
4056            self.settle_questions();
4057            bail!("no candidate produced a change; nothing to judge");
4058        }
4059        self.state.status = RunStatus::Judging;
4060        self.state.save()?;
4061        Ok(())
4062    }
4063
4064    // --------------------------------------------------------------- judge
4065
4066    async fn judge(&mut self) -> Result<()> {
4067        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4068        // agent files with `magi task add` name the run that paid for it. The
4069        // prompt overlay is cloned alongside it because the waves borrow it
4070        // while `self` is mutably borrowed by the node's own bookkeeping.
4071        let run_id = self.state.id.clone();
4072        let prompts = self.state.config.prompts.clone();
4073        if !self.state.judgements.is_empty() || self.state.judge_skipped {
4074            return Ok(());
4075        }
4076        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4077        if viable.len() == 1 {
4078            // Recorded so this is a one-time event: `judgements` stays empty
4079            // either way, which without this flag is indistinguishable from
4080            // "not yet judged" on the next reentry — and status is left
4081            // untouched, so a later node's conclusion (e.g. `Blocked` after
4082            // the review budget ran out) survives a resume instead of being
4083            // clobbered back to `Judging` by this node running again.
4084            self.state.judge_skipped = true;
4085            self.state.event(
4086                "judge",
4087                format!(
4088                    "only candidate {} produced a change; judging skipped",
4089                    viable[0].label
4090                ),
4091            );
4092            self.state.save()?;
4093            return Ok(());
4094        }
4095        self.state.status = RunStatus::Judging;
4096
4097        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4098        let language = self.state.config.graph.language.clone();
4099        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4100        let sessions = self.state.config.graph.sessions;
4101        let artifacts = agent::artifacts_dir(&self.state.dir());
4102        let root = self.state.worktree_root();
4103        let base_short = short(&self.state.base_commit);
4104
4105        let mut jobs = Vec::new();
4106        let mut orders = Vec::new();
4107        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4108            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4109            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4110            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4111            let seat_key = format!("judge-{}", j + 1);
4112            let seat = self.seat(&seat_key, &spec.id);
4113            jobs.push(SeatJob {
4114                prompt: prompt::judge(
4115                    &self.state.instruction,
4116                    &views,
4117                    self.roles.judges.len(),
4118                    &base_short,
4119                    &language,
4120                ),
4121                spec,
4122                seat,
4123                cwd: root.join(format!("judge-{}", j + 1)),
4124                timeout,
4125                allow_write: false,
4126                sessions,
4127                artifacts: artifacts.clone(),
4128                stem: format!("judge-{}", j + 1),
4129                handover: None,
4130            });
4131        }
4132
4133        self.state.event(
4134            "judge",
4135            format!(
4136                "{} judges ranking {} candidates blind",
4137                jobs.len(),
4138                viable.len()
4139            ),
4140        );
4141        let labels_for_check = labels.clone();
4142        let mut quota_losses = Vec::new();
4143        let cache = self.state.config.cache_dir();
4144        let ctx = WaveCtx {
4145            carry_seats: false,
4146            pauses: Some((&self.pause, &self.interrupt)),
4147            run: &run_id,
4148            node: "judge",
4149            prompts: &prompts,
4150            cache: cache.as_deref(),
4151            round: None,
4152        };
4153        let results = ask_json_wave::<Ranking>(
4154            jobs,
4155            Arc::clone(&self.sem),
4156            self.state.config.graph.retries,
4157            &self.roles.judge_roster,
4158            &ctx,
4159            &mut quota_losses,
4160            &mut self.state,
4161            &move |r: &Ranking| r.validate(&labels_for_check),
4162        )
4163        .await;
4164        if self.state.resource_block.is_some() {
4165            return self.stop_for_resource();
4166        }
4167        self.state.quota.extend(quota_losses);
4168
4169        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4170            let agent_id = seat.agent.clone();
4171            self.state.seats.insert(seat.key.clone(), seat);
4172            let mut record = Judgement {
4173                judge: j + 1,
4174                seat: format!("judge-{}", j + 1),
4175                agent: agent_id,
4176                ranking: Vec::new(),
4177                reasons: BTreeMap::new(),
4178                confidence: None,
4179                strengths: BTreeMap::new(),
4180                order: orders[j].clone(),
4181                failed: None,
4182                duration_ms: 0,
4183            };
4184            match res {
4185                Ok((ranking, out)) => {
4186                    record.ranking = ranking.normalized();
4187                    record.reasons = ranking.reasons;
4188                    record.strengths = ranking.strengths;
4189                    record.confidence = ranking.confidence;
4190                    record.duration_ms = out.duration_ms;
4191                    self.state.event(
4192                        "judge",
4193                        format!(
4194                            "judge {} ranked {}",
4195                            j + 1,
4196                            record.ranking.iter().collect::<String>()
4197                        ),
4198                    );
4199                }
4200                Err(e) => {
4201                    record.failed = Some(e.to_string());
4202                    self.state
4203                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4204                }
4205            }
4206            self.state.judgements.push(record);
4207            self.state.save()?;
4208        }
4209        Ok(())
4210    }
4211
4212    // ---------------------------------------------------------- deliberate
4213
4214    async fn deliberate(&mut self) -> Result<()> {
4215        if self.hold_base_sync().await? {
4216            return Ok(());
4217        }
4218        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4219        // agent files with `magi task add` name the run that paid for it. The
4220        // prompt overlay is cloned alongside it because the waves borrow it
4221        // while `self` is mutably borrowed by the node's own bookkeeping.
4222        let run_id = self.state.id.clone();
4223        let prompts = self.state.config.prompts.clone();
4224        let rounds = self.state.config.graph.deliberate_rounds;
4225        // Resume unfinished rounds after an outage, but never reopen a tally.
4226        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4227            return Ok(());
4228        }
4229        let tops: Vec<char> = self
4230            .state
4231            .judgements
4232            .iter()
4233            .filter_map(|j| j.ranking.first().copied())
4234            .collect();
4235        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4236            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4237                self.state.event(
4238                    "deliberate",
4239                    format!("judges agreed on {} outright; no deliberation", tops[0]),
4240                );
4241            }
4242            self.state.status = RunStatus::Voting;
4243            self.state.save()?;
4244            return Ok(());
4245        }
4246
4247        self.state.status = RunStatus::Deliberating;
4248        self.state.event(
4249            "deliberate",
4250            format!(
4251                "split: first choices were {} — opening {rounds} round(s)",
4252                tops.iter().collect::<String>()
4253            ),
4254        );
4255
4256        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4257        let language = self.state.config.graph.language.clone();
4258        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4259        let sessions = self.state.config.graph.sessions;
4260        let artifacts = agent::artifacts_dir(&self.state.dir());
4261        let root = self.state.worktree_root();
4262        let base_short = short(&self.state.base_commit);
4263
4264        // Judges argue in sequence so that a turn can answer the one before it;
4265        // that is the difference between deliberation and three parallel
4266        // monologues.
4267        for round in self.state.deliberation.len() + 1..=rounds {
4268            let mut turns: Vec<DeliberationTurn> = Vec::new();
4269            let mut round_losses = Vec::new();
4270            let mut asked = 0;
4271            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4272                if self.state.judgements[j].failed.is_some() {
4273                    continue;
4274                }
4275                let seat_key = format!("judge-{}", j + 1);
4276                let spec = self.occupant(&seat_key, spec);
4277                let mut seat = self.seat(&seat_key, &spec.id);
4278                let transcript = self.transcript(&turns, j);
4279                let build = |context: Option<&str>| {
4280                    prompt::deliberate(
4281                        &self.state.instruction,
4282                        context,
4283                        &transcript,
4284                        round,
4285                        rounds,
4286                        &language,
4287                    )
4288                };
4289                let block = self.candidate_block(&viable, &base_short);
4290                let full = build(Some(&block));
4291                let text = if has_context(&spec, &seat, sessions) {
4292                    build(None)
4293                } else {
4294                    full.clone()
4295                };
4296                let job = SeatJob {
4297                    spec,
4298                    seat: seat.clone(),
4299                    prompt: text,
4300                    cwd: root.join(format!("judge-{}", j + 1)),
4301                    timeout,
4302                    allow_write: false,
4303                    sessions,
4304                    artifacts: artifacts.clone(),
4305                    stem: format!("delib-{round}-judge-{}", j + 1),
4306                    handover: Some(full),
4307                };
4308                let cache = self.state.config.cache_dir();
4309                let ctx = WaveCtx {
4310                    carry_seats: false,
4311                    pauses: Some((&self.pause, &self.interrupt)),
4312                    run: &run_id,
4313                    node: "deliberate",
4314                    prompts: &prompts,
4315                    cache: cache.as_deref(),
4316                    round: None,
4317                };
4318                // A turn is never nudged (`retries` 0): a failed seat is
4319                // handed to the next roster agent, which gets the full
4320                // context. An empty answer is a turn, not a failure.
4321                let mut losses = Vec::new();
4322                asked += 1;
4323                let mut results = ask_wave_with::<String>(
4324                    vec![job],
4325                    Arc::clone(&self.sem),
4326                    0,
4327                    &self.roles.judge_roster,
4328                    &ctx,
4329                    &mut losses,
4330                    &mut self.state,
4331                    &|text: &str| {
4332                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4333                    },
4334                )
4335                .await;
4336                if self.state.resource_block.is_some() {
4337                    return self.stop_for_resource();
4338                }
4339                round_losses.extend(losses.iter().cloned());
4340                self.state.quota.extend(losses);
4341                let (updated, res, _) = results.pop().expect("one job in, one result out");
4342                seat = updated;
4343                let agent_id = seat.agent.clone();
4344                self.state.seats.insert(seat.key.clone(), seat);
4345                let body = match res {
4346                    Ok((body, _)) => body,
4347                    // Skip the seat; a CLI's raw error JSON is never read as
4348                    // this judge's position.
4349                    Err(e) => {
4350                        self.state
4351                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4352                        continue;
4353                    }
4354                };
4355                let tentative = verdict::extract_json::<Position>(&body)
4356                    .ok()
4357                    .and_then(|p| p.tentative)
4358                    .and_then(|s| s.trim().chars().next())
4359                    .map(|c| c.to_ascii_uppercase());
4360                self.state.event(
4361                    "deliberate",
4362                    format!(
4363                        "round {round}: judge {} now favours {}",
4364                        j + 1,
4365                        tentative.map_or("—".to_owned(), |c| c.to_string())
4366                    ),
4367                );
4368                turns.push(DeliberationTurn {
4369                    judge: j + 1,
4370                    agent: agent_id,
4371                    body: blind::sanitize_prose(&body, &self.state.config.blind),
4372                    tentative,
4373                });
4374            }
4375            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4376                block_machine_outage(&mut self.state, "deliberate", &round_losses);
4377                return self.stop_for_resource();
4378            }
4379            self.state
4380                .deliberation
4381                .push(DeliberationRound { round, turns });
4382            self.state.save()?;
4383        }
4384
4385        self.state.status = RunStatus::Voting;
4386        self.state.save()?;
4387        Ok(())
4388    }
4389
4390    // ---------------------------------------------------------------- vote
4391
4392    async fn vote(&mut self) -> Result<()> {
4393        if self.hold_base_sync().await? {
4394            return Ok(());
4395        }
4396        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4397        // agent files with `magi task add` name the run that paid for it. The
4398        // prompt overlay is cloned alongside it because the waves borrow it
4399        // while `self` is mutably borrowed by the node's own bookkeeping.
4400        let run_id = self.state.id.clone();
4401        let prompts = self.state.config.prompts.clone();
4402        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4403            return Ok(());
4404        }
4405        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4406        if viable.len() == 1 {
4407            return Ok(());
4408        }
4409        self.state.status = RunStatus::Voting;
4410
4411        let language = self.state.config.graph.language.clone();
4412        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4413        let sessions = self.state.config.graph.sessions;
4414        let artifacts = agent::artifacts_dir(&self.state.dir());
4415        let root = self.state.worktree_root();
4416        let base_short = short(&self.state.base_commit);
4417        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4418
4419        let mut jobs = Vec::new();
4420        let mut seats_at = Vec::new();
4421        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4422            if self
4423                .state
4424                .judgements
4425                .get(j)
4426                .is_some_and(|r| r.failed.is_some())
4427            {
4428                continue;
4429            }
4430            let seat_key = format!("judge-{}", j + 1);
4431            let spec = self.occupant(&seat_key, spec);
4432            let seat = self.seat(&seat_key, &spec.id);
4433            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4434            let text = if has_context(&spec, &seat, sessions) {
4435                prompt::final_vote(&viable, &language)
4436            } else {
4437                full.clone()
4438            };
4439            jobs.push(SeatJob {
4440                spec,
4441                seat,
4442                prompt: text,
4443                cwd: root.join(format!("judge-{}", j + 1)),
4444                timeout,
4445                allow_write: false,
4446                sessions,
4447                artifacts: artifacts.clone(),
4448                stem: format!("vote-judge-{}", j + 1),
4449                handover: Some(full),
4450            });
4451            seats_at.push(j);
4452        }
4453
4454        self.state.event(
4455            "vote",
4456            format!(
4457                "collecting {} final votes one by one, privately",
4458                jobs.len()
4459            ),
4460        );
4461        let allowed = viable.clone();
4462        let mut quota_losses = Vec::new();
4463        let cache = self.state.config.cache_dir();
4464        let ctx = WaveCtx {
4465            carry_seats: false,
4466            pauses: Some((&self.pause, &self.interrupt)),
4467            run: &run_id,
4468            node: "vote",
4469            prompts: &prompts,
4470            cache: cache.as_deref(),
4471            round: None,
4472        };
4473        let results = ask_json_wave::<FinalVote>(
4474            jobs,
4475            Arc::clone(&self.sem),
4476            self.state.config.graph.retries,
4477            &self.roles.judge_roster,
4478            &ctx,
4479            &mut quota_losses,
4480            &mut self.state,
4481            &move |v: &FinalVote| match v.label() {
4482                Some(c) if allowed.contains(&c) => Ok(()),
4483                other => bail!("vote {other:?} is not one of {allowed:?}"),
4484            },
4485        )
4486        .await;
4487        if self.state.resource_block.is_some() {
4488            return self.stop_for_resource();
4489        }
4490        self.state.quota.extend(quota_losses);
4491
4492        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4493            let agent_id = seat.agent.clone();
4494            self.state.seats.insert(seat.key.clone(), seat);
4495            let initial = self
4496                .state
4497                .judgements
4498                .get(j)
4499                .and_then(|r| r.ranking.first().copied());
4500            let mut record = VoteRecord {
4501                judge: j + 1,
4502                agent: agent_id,
4503                vote: None,
4504                reason: String::new(),
4505                changed: false,
4506            };
4507            match res {
4508                Ok((v, _)) => {
4509                    record.vote = v.label();
4510                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4511                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4512                    self.state.event(
4513                        "vote",
4514                        format!(
4515                            "judge {} voted {}{}",
4516                            j + 1,
4517                            record.vote.unwrap_or('?'),
4518                            if record.changed { " (changed)" } else { "" }
4519                        ),
4520                    );
4521                }
4522                Err(e) => {
4523                    self.state
4524                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4525                }
4526            }
4527            self.state.votes.push(record);
4528            self.state.save()?;
4529        }
4530        Ok(())
4531    }
4532
4533    // --------------------------------------------------------------- tally
4534
4535    fn tally(&mut self) -> Result<()> {
4536        if self.state.tally.is_some() {
4537            return Ok(());
4538        }
4539        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4540        let tops: Vec<char> = self
4541            .state
4542            .judgements
4543            .iter()
4544            .filter_map(|j| j.ranking.first().copied())
4545            .collect();
4546        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4547
4548        // A judge whose private vote failed still counted once, in the initial
4549        // ranking; using it beats discarding a whole seat.
4550        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4551        let mut cast: Vec<char> = Vec::new();
4552        for (i, j) in self.state.judgements.iter().enumerate() {
4553            let vote = self
4554                .state
4555                .votes
4556                .iter()
4557                .find(|v| v.judge == i + 1)
4558                .and_then(|v| v.vote)
4559                .or_else(|| j.ranking.first().copied());
4560            if let Some(v) = vote {
4561                *first_choice.entry(v).or_insert(0) += 1;
4562                cast.push(v);
4563            }
4564        }
4565
4566        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4567        for j in &self.state.judgements {
4568            let n = j.ranking.len();
4569            for (pos, label) in j.ranking.iter().enumerate() {
4570                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4571            }
4572        }
4573
4574        let best = first_choice.values().copied().max().unwrap_or(0);
4575        let mut leaders: Vec<char> = first_choice
4576            .iter()
4577            .filter(|(_, v)| **v == best)
4578            .map(|(k, _)| *k)
4579            .collect();
4580        let mut tie_break = None;
4581        if leaders.len() > 1 {
4582            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4583            let borda_leaders: Vec<char> = leaders
4584                .iter()
4585                .copied()
4586                .filter(|l| borda[l] == top_borda)
4587                .collect();
4588            tie_break = Some(if borda_leaders.len() == 1 {
4589                format!(
4590                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4591                    leaders.len()
4592                )
4593            } else {
4594                format!(
4595                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4596                    leaders.len()
4597                )
4598            });
4599            leaders = borda_leaders;
4600            leaders.sort_unstable();
4601        }
4602        let winner = *leaders
4603            .first()
4604            .or(viable.first())
4605            .context("no candidate to declare a winner from")?;
4606
4607        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4608        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4609        let deliberated = !self.state.deliberation.is_empty();
4610
4611        // Whose verdict is this? A rate-limited seat is absent even if it
4612        // ranked before the limit hit, so presence is measured against the
4613        // recorded losses, not just "did a ranking ever appear".
4614        let quota_seats: std::collections::BTreeSet<&str> =
4615            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4616        let mut present = 0usize;
4617        for (i, j) in self.state.judgements.iter().enumerate() {
4618            if quota_seats.contains(j.seat.as_str()) {
4619                continue;
4620            }
4621            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4622            let voted = self
4623                .state
4624                .votes
4625                .iter()
4626                .any(|v| v.judge == i + 1 && v.vote.is_some());
4627            if ranked || voted {
4628                present += 1;
4629            }
4630        }
4631        // Strict majority of the configured panel. A bare majority is real
4632        // signal we can act on, while a minority verdict must never stand in
4633        // for a healthy one. A one-candidate run needs no panel at all, and
4634        // `judges` stays `0` rather than the roster size a panel that never
4635        // sat would otherwise be credited with.
4636        let needs_quorum = viable.len() > 1;
4637        let judges_total = if needs_quorum {
4638            self.roles.judges.len()
4639        } else {
4640            0
4641        };
4642        let quorum = if needs_quorum {
4643            judges_total / 2 + 1
4644        } else {
4645            0
4646        };
4647        let met_quorum = !needs_quorum || present >= quorum;
4648        let uncontested = (!needs_quorum).then(|| {
4649            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4650        });
4651
4652        self.state.event(
4653            "tally",
4654            match &uncontested {
4655                Some(reason) => format!("winner {winner} — {reason}"),
4656                None => format!(
4657                    "winner {winner} — votes {} | initial {} | {} changed | \
4658                     {present}/{judges_total} judges{}",
4659                    first_choice
4660                        .iter()
4661                        .map(|(k, v)| format!("{k}:{v}"))
4662                        .collect::<Vec<_>>()
4663                        .join(" "),
4664                    if unanimous_initial {
4665                        "unanimous"
4666                    } else {
4667                        "split"
4668                    },
4669                    changed_votes,
4670                    if met_quorum {
4671                        String::new()
4672                    } else {
4673                        format!(" — below quorum ({quorum} required)")
4674                    },
4675                ),
4676            },
4677        );
4678        if !met_quorum {
4679            self.state.event(
4680                "stall",
4681                format!(
4682                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4683                     the run stops here, resumable"
4684                ),
4685            );
4686        }
4687        self.state.tally = Some(Tally {
4688            first_choice,
4689            borda,
4690            winner,
4691            rankings: tops.len(),
4692            unanimous_initial,
4693            deliberated,
4694            changed_votes,
4695            unanimous_final,
4696            tie_break,
4697            judges: judges_total,
4698            present,
4699            quorum,
4700            met_quorum,
4701            uncontested,
4702        });
4703        self.state.status = if met_quorum {
4704            RunStatus::Reviewing
4705        } else {
4706            RunStatus::Stalled
4707        };
4708        self.state.save()?;
4709        Ok(())
4710    }
4711
4712    // ------------------------------------------------------------- recover
4713
4714    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4715    /// resumed toward completion once the transient cause clears.
4716    ///
4717    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4718    /// it toward the quorum, which is exactly the set of seats whose absence
4719    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4720    /// not depend on which node happened to hit the limit), or an ordinary
4721    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4722    /// seat is never disturbed.
4723    ///
4724    /// A seat that now answers with a usable ranking is "recovered": its
4725    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4726    /// `tally` counts it present again), and its vote re-collected. A seat that
4727    /// still fails keeps its loss and stays absent.
4728    ///
4729    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4730    /// to review/gate/merge), `false` when it is still below quorum (the run
4731    /// stays `Stalled`, still resumable for a later retry).
4732    #[allow(clippy::too_many_lines)]
4733    async fn recover_stall(&mut self) -> Result<bool> {
4734        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4735        // agent files with `magi task add` name the run that paid for it. The
4736        // prompt overlay is cloned alongside it because the waves borrow it
4737        // while `self` is mutably borrowed by the node's own bookkeeping.
4738        let run_id = self.state.id.clone();
4739        let prompts = self.state.config.prompts.clone();
4740        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4741        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4742        // is what keeps a non-quota collapse — or a quota loss recorded at the
4743        // deliberate node — from being a permanent dead-end on `--resume`.
4744        let quota_seats: BTreeSet<&str> =
4745            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4746        let absent: Vec<String> = self
4747            .state
4748            .judgements
4749            .iter()
4750            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4751            .map(|j| j.seat.clone())
4752            .collect();
4753        if absent.is_empty() {
4754            return Ok(false);
4755        }
4756        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4757        if viable.len() <= 1 {
4758            return Ok(false);
4759        }
4760        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4761        let language = self.state.config.graph.language.clone();
4762        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4763        let sessions = self.state.config.graph.sessions;
4764        let artifacts = agent::artifacts_dir(&self.state.dir());
4765        let root = self.state.worktree_root();
4766        let base_short = short(&self.state.base_commit);
4767        let candidates: Vec<Candidate> = viable.clone();
4768
4769        // Map each absent seat key to its 0-based position in `roles.judges`.
4770        let mut positions: Vec<usize> = absent
4771            .iter()
4772            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4773            .collect();
4774        if positions.is_empty() {
4775            return Ok(false);
4776        }
4777        positions.sort_unstable();
4778        positions.dedup();
4779
4780        // Re-rank the lost seats, one blind prompt each.
4781        let mut judge_jobs = Vec::new();
4782        for &j in &positions {
4783            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4784            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4785            let seat_key = format!("judge-{}", j + 1);
4786            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4787            let seat = self.seat(&seat_key, &spec.id);
4788            judge_jobs.push(SeatJob {
4789                spec,
4790                seat,
4791                prompt: prompt::judge(
4792                    &self.state.instruction,
4793                    &views,
4794                    self.roles.judges.len(),
4795                    &base_short,
4796                    &language,
4797                ),
4798                cwd: root.join(seat_key),
4799                timeout,
4800                allow_write: false,
4801                sessions,
4802                artifacts: artifacts.clone(),
4803                stem: format!("judge-{}-recover", j + 1),
4804                handover: None,
4805            });
4806        }
4807
4808        let labels_for_check = labels.clone();
4809        let mut judge_losses = Vec::new();
4810        let retries = self.state.config.graph.retries;
4811        let cache = self.state.config.cache_dir();
4812        let ctx = WaveCtx {
4813            carry_seats: false,
4814            pauses: Some((&self.pause, &self.interrupt)),
4815            run: &run_id,
4816            node: "judge",
4817            prompts: &prompts,
4818            cache: cache.as_deref(),
4819            round: None,
4820        };
4821        let results = ask_json_wave::<Ranking>(
4822            judge_jobs,
4823            Arc::clone(&self.sem),
4824            retries,
4825            &self.roles.judge_roster,
4826            &ctx,
4827            &mut judge_losses,
4828            &mut self.state,
4829            &move |r: &Ranking| r.validate(&labels_for_check),
4830        )
4831        .await;
4832        if self.state.resource_block.is_some() {
4833            self.stop_for_resource()?;
4834            return Ok(false);
4835        }
4836
4837        // Refresh the judgement of every seat that ranked again.
4838        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4839        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4840            let agent_id = seat.agent.clone();
4841            self.state.seats.insert(seat.key.clone(), seat);
4842            let record = &mut self.state.judgements[j];
4843            match res {
4844                Ok((ranking, out)) => {
4845                    record.agent = agent_id;
4846                    record.ranking = ranking.normalized();
4847                    record.reasons = ranking.reasons;
4848                    record.strengths = ranking.strengths;
4849                    record.confidence = ranking.confidence;
4850                    record.failed = None;
4851                    record.duration_ms = out.duration_ms;
4852                    recovered.insert(j);
4853                    self.state.event(
4854                        "recover",
4855                        format!("judge {} ranked again after the limit", j + 1),
4856                    );
4857                }
4858                Err(e) => {
4859                    self.state
4860                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4861                }
4862            }
4863        }
4864
4865        // Re-ask the votes of the seats that recovered a ranking.
4866        let mut vote_jobs = Vec::new();
4867        let mut vote_pos: Vec<usize> = Vec::new();
4868        for &j in &recovered {
4869            let seat_key = format!("judge-{}", j + 1);
4870            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4871            let seat = self.seat(&seat_key, &spec.id);
4872            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4873            let text = if has_context(&spec, &seat, sessions) {
4874                prompt::final_vote(&labels, &language)
4875            } else {
4876                full.clone()
4877            };
4878            vote_jobs.push(SeatJob {
4879                spec,
4880                seat,
4881                prompt: text,
4882                cwd: root.join(seat_key),
4883                timeout,
4884                allow_write: false,
4885                sessions,
4886                artifacts: artifacts.clone(),
4887                stem: format!("vote-judge-{}-recover", j + 1),
4888                handover: Some(full),
4889            });
4890            vote_pos.push(j);
4891        }
4892        let allowed = labels.clone();
4893        let mut vote_losses = Vec::new();
4894        let vote_retries = self.state.config.graph.retries;
4895        let vote_cache = self.state.config.cache_dir();
4896        let ctx = WaveCtx {
4897            carry_seats: false,
4898            pauses: Some((&self.pause, &self.interrupt)),
4899            run: &run_id,
4900            node: "vote",
4901            prompts: &prompts,
4902            cache: vote_cache.as_deref(),
4903            round: None,
4904        };
4905        let votes = ask_json_wave::<FinalVote>(
4906            vote_jobs,
4907            Arc::clone(&self.sem),
4908            vote_retries,
4909            &self.roles.judge_roster,
4910            &ctx,
4911            &mut vote_losses,
4912            &mut self.state,
4913            &move |v: &FinalVote| match v.label() {
4914                Some(c) if allowed.contains(&c) => Ok(()),
4915                other => bail!("vote {other:?} is not one of {allowed:?}"),
4916            },
4917        )
4918        .await;
4919        if self.state.resource_block.is_some() {
4920            self.stop_for_resource()?;
4921            return Ok(false);
4922        }
4923        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4924            let agent_id = seat.agent.clone();
4925            self.state.seats.insert(seat.key.clone(), seat);
4926            match res {
4927                Ok((v, _)) => {
4928                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4929                        rec.vote = v.label();
4930                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4931                    } else {
4932                        self.state.votes.push(VoteRecord {
4933                            judge: j + 1,
4934                            agent: agent_id,
4935                            vote: v.label(),
4936                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4937                            changed: false,
4938                        });
4939                    }
4940                    self.state.event(
4941                        "recover",
4942                        format!("judge {} voted again after the limit", j + 1),
4943                    );
4944                }
4945                Err(e) => {
4946                    self.state
4947                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4948                }
4949            }
4950        }
4951
4952        // A seat that ranked again is present even if its re-vote failed —
4953        // `tally` falls back to the initial ranking's first choice — so clear
4954        // its quota loss. Seats that still fail keep theirs and stay absent.
4955        let recovered_keys: BTreeSet<String> = recovered
4956            .iter()
4957            .map(|&j| format!("judge-{}", j + 1))
4958            .collect();
4959        self.state
4960            .quota
4961            .retain(|q| !recovered_keys.contains(&q.seat));
4962        // A seat that hit the limit again is a fresh loss, not the old one:
4963        // replace the stale entry so the history stays one-per-seat and the
4964        // daemon can tell this attempt's loss from a previous session's.
4965        for loss in judge_losses.into_iter().chain(vote_losses) {
4966            if recovered_keys.contains(&loss.seat) {
4967                continue;
4968            }
4969            self.state.quota.retain(|q| q.seat != loss.seat);
4970            self.state.quota.push(loss);
4971        }
4972
4973        // Recompute the verdict from the refreshed panel.
4974        self.state.tally = None;
4975        self.tally()?;
4976        Ok(self
4977            .state
4978            .tally
4979            .as_ref()
4980            .map(|t| t.met_quorum)
4981            .unwrap_or(false))
4982    }
4983
4984    // ---------------------------------------------------------------- graft
4985
4986    /// Port the strengths the judges attributed to the losing candidates onto
4987    /// the winner, once, between the tally and the review loop.
4988    ///
4989    /// Best-effort by construction: it returns nothing and fails nothing. A
4990    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4991    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4992    /// the run carries on as if the phase were off. See [`graft`] for the
4993    /// decisions and `AGENTS.md` for the invariants.
4994    ///
4995    /// The one exception is a rollback that could not be confirmed: the
4996    /// rejected commit may still be the winner's tip, so the run is blocked
4997    /// (resumable; the record stays `Running`, which makes a resume retry the
4998    /// restore) instead of reviewing, gating and merging it.
4999    async fn graft(&mut self) -> Result<()> {
5000        let Err(e) = self.graft_inner().await else {
5001            return Ok(());
5002        };
5003        if e.is::<crate::repo_lock::ResourceBlocked>() {
5004            return Err(e);
5005        }
5006        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5007            self.state.status = RunStatus::Blocked;
5008            self.state.event("graft", format!("blocked: {e:#}"));
5009            self.state.save()?;
5010            return Err(e);
5011        }
5012        // An error after the `Running` record was saved may have left the
5013        // fixer's work in the tree: the record must not stay unsettled
5014        // (a later resume would reset past the review fixes), so confirm the
5015        // restore now, or block.
5016        if let Some(rec) = self.state.graft.clone()
5017            && !rec.outcome.settled()
5018        {
5019            if let Some(winner) = self.state.winner().cloned() {
5020                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5021                    if re.is::<crate::repo_lock::ResourceBlocked>() {
5022                        return Err(re);
5023                    }
5024                    self.state.status = RunStatus::Blocked;
5025                    self.state.event("graft", format!("blocked: {re:#}"));
5026                    self.state.save()?;
5027                    return Err(re);
5028                }
5029            }
5030            if let Some(g) = self.state.graft.as_mut() {
5031                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5032            }
5033        }
5034        self.state.event("graft", format!("skipped: {e:#}"));
5035        if let Err(e) = self.state.save() {
5036            tracing::warn!("save after graft: {e:#}");
5037        }
5038        Ok(())
5039    }
5040
5041    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
5042    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
5043    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5044        let _coordination = self.coordinate(wt).await?;
5045        crate::recovery::restore_repair(wt, pre)
5046            .await
5047            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5048    }
5049
5050    /// A skip is a run event, recorded once however often the node is re-entered.
5051    fn graft_skipped(&mut self, why: &str) {
5052        let message = format!("skipped: {why}");
5053        if !self
5054            .state
5055            .events
5056            .iter()
5057            .any(|e| e.node == "graft" && e.message == message)
5058        {
5059            self.state.event("graft", message);
5060        }
5061    }
5062
5063    async fn graft_inner(&mut self) -> Result<()> {
5064        // A round that was started and never finished (crash, kill): the
5065        // fixer may have left anything in the tree, so put the winner back.
5066        if let Some(rec) = self.state.graft.clone()
5067            && !rec.outcome.settled()
5068        {
5069            // Reviews exist only if the graft ended without its record being
5070            // settled (a swallowed error): the tree has moved on since, and
5071            // a reset to `pre` would discard the review fixes.
5072            if self.state.reviews.is_empty()
5073                && let Some(winner) = self.state.winner().cloned()
5074            {
5075                self.graft_restore(&winner.worktree, &rec.pre).await?;
5076            }
5077            if rec.outcome == GraftOutcome::WaitingForRestore {
5078                self.state.graft = None;
5079                self.state.save()?;
5080            } else {
5081                if let Some(g) = self.state.graft.as_mut() {
5082                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5083                }
5084                self.state.event(
5085                    "graft",
5086                    "rolled back: the previous round was interrupted; the winner is untouched",
5087                );
5088                self.state.save()?;
5089                return Ok(());
5090            }
5091        }
5092        let plan = match graft::plan(&self.state) {
5093            Ok(plan) => plan,
5094            Err(skip) => {
5095                if skip != graft::Skip::AlreadyDone {
5096                    self.graft_skipped(skip.reason());
5097                    self.state.save()?;
5098                }
5099                return Ok(());
5100            }
5101        };
5102        let Some(winner) = self.state.winner().cloned() else {
5103            return Ok(());
5104        };
5105        let wt = winner.worktree.clone();
5106        if !git::is_clean(&wt).await? {
5107            self.graft_skipped("the winner's worktree has uncommitted changes");
5108            self.state.save()?;
5109            return Ok(());
5110        }
5111        let pre = git::rev_parse(&wt, "HEAD").await?;
5112        // Saved before the fixer is called: a stop from here on is repaired
5113        // by the branch above on the next entry.
5114        self.state.graft = Some(GraftRecord {
5115            pre: pre.clone(),
5116            post: None,
5117            lines: None,
5118            limit: None,
5119            agent: None,
5120            outcome: GraftOutcome::Running,
5121            requested: graft::requested(&plan),
5122            adoption: None,
5123            adoption_error: None,
5124        });
5125        self.state.event(
5126            "graft",
5127            format!(
5128                "porting strengths of candidate(s) {} onto {}",
5129                plan.strengths
5130                    .iter()
5131                    .map(|(l, _, _)| l.to_string())
5132                    .collect::<Vec<_>>()
5133                    .join(", "),
5134                plan.winner
5135            ),
5136        );
5137        self.state.save()?;
5138
5139        let fix_prompt = prompt::graft(
5140            &self.state.instruction,
5141            plan.winner,
5142            &plan.strengths,
5143            &self.state.config.graph.language,
5144        );
5145        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5146        let sessions = self.state.config.graph.sessions;
5147        let artifacts = agent::artifacts_dir(&self.state.dir());
5148        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5149        let (_, seat, out) = self
5150            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5151                SeatJob {
5152                    prompt: fix_prompt.clone(),
5153                    spec,
5154                    seat,
5155                    cwd: wt.clone(),
5156                    timeout,
5157                    allow_write: true,
5158                    sessions,
5159                    artifacts: artifacts.clone(),
5160                    stem: "graft".to_owned(),
5161                    handover: None,
5162                }
5163            })
5164            .await;
5165        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5166            // Preserve the need to restore earlier fallback edits without
5167            // confusing an unstarted seat with an interrupted invocation.
5168            self.state.graft.as_mut().expect("graft record").outcome =
5169                GraftOutcome::WaitingForRestore;
5170            return self.stop_for_resource();
5171        }
5172        let mut why: Option<String> = None;
5173        match &out {
5174            AgentOutcome::Ok(o) if agent::output_advances(o) => {
5175                why = Some("the fixer produced nothing usable".to_owned());
5176            }
5177            AgentOutcome::Ok(_) => {}
5178            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5179            AgentOutcome::Quota(o) => {
5180                self.state.quota.push(QuotaLoss::from_report(
5181                    seat.key.clone(),
5182                    "graft",
5183                    o.quota.as_ref(),
5184                ));
5185                why = Some("rate limited (quota)".to_owned());
5186            }
5187            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5188                why = Some(format!(
5189                    "the fixer failed: {}",
5190                    e.lines()
5191                        .next()
5192                        .unwrap_or("")
5193                        .chars()
5194                        .take(160)
5195                        .collect::<String>()
5196                ));
5197            }
5198        }
5199        let agent_id = seat.agent.clone();
5200        self.state.seats.insert(seat.key.clone(), seat);
5201
5202        let mut post = None;
5203        let mut lines = None;
5204        let _coordination = self.coordinate(&wt).await?;
5205        if why.is_none() {
5206            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5207            if let Ok(r) = &rescued {
5208                self.state.note_withheld("graft", &r.withheld);
5209            }
5210            let head = git::rev_parse(&wt, "HEAD").await?;
5211            // What is measured must be what gets reviewed and gated: a failed
5212            // rescue, or anything still uncommitted (withheld lockfiles
5213            // aside, which are never part of the tree), rejects the graft.
5214            let dirty = match &rescued {
5215                Ok(r) => git::status_porcelain(&wt)
5216                    .await
5217                    .map(|st| {
5218                        st.lines().any(|l| {
5219                            !r.withheld
5220                                .iter()
5221                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
5222                        })
5223                    })
5224                    .unwrap_or(true),
5225                Err(_) => true,
5226            };
5227            if dirty {
5228                why = Some("the fixer left changes that could not be committed".to_owned());
5229            } else if head == pre {
5230                why = Some("the fixer changed nothing".to_owned());
5231            } else {
5232                lines = git::changed_lines(&wt, &pre, &head).await;
5233                post = Some(head);
5234            }
5235        }
5236        let (adoption, adoption_error) = match &out {
5237            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5238            _ => (None, None),
5239        };
5240        let requested = graft::requested(&plan);
5241        let outcome = match why {
5242            None => {
5243                self.state.event(
5244                    "graft",
5245                    format!(
5246                        "applied: {}",
5247                        lines.map_or_else(
5248                            || "changed lines unavailable".to_owned(),
5249                            |n| format!("{n} changed lines")
5250                        )
5251                    ),
5252                );
5253                GraftOutcome::Applied
5254            }
5255            Some(why) => {
5256                // Not settled until the restore is confirmed: the record
5257                // stays `Running`, so a resume retries it.
5258                crate::recovery::restore_repair(&wt, &pre)
5259                    .await
5260                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5261                self.state.event(
5262                    "graft",
5263                    format!("rolled back: {why}; the winner is untouched"),
5264                );
5265                GraftOutcome::RolledBack(why)
5266            }
5267        };
5268        self.state.graft = Some(GraftRecord {
5269            pre,
5270            post,
5271            lines,
5272            limit: None,
5273            agent: Some(agent_id),
5274            outcome,
5275            requested,
5276            adoption,
5277            adoption_error,
5278        });
5279        self.state.save()?;
5280        Ok(())
5281    }
5282
5283    // ----------------------------------------------------------------- fold
5284
5285    async fn fold_losers(&mut self) -> Result<()> {
5286        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5287            return Ok(());
5288        };
5289        if !self.state.candidate_preservation.is_empty() {
5290            bail!("candidate preservation incomplete; refusing cleanup");
5291        }
5292        let repo = self.state.repo.clone();
5293        let _coordination = self.coordinate(&repo).await?;
5294        let mut folded = Vec::new();
5295        for i in 0..self.state.candidates.len() {
5296            let c = &self.state.candidates[i];
5297            if c.label == winner || c.folded {
5298                continue;
5299            }
5300            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5301            remove_preserved_worktree(&repo, &wt).await?;
5302            if git::branch_exists(&repo, &branch).await?
5303                && !git::branch_delete(&repo, &branch).await?
5304            {
5305                bail!("candidate branch removal refused");
5306            }
5307            self.state.candidates[i].folded = true;
5308            self.state.save()?;
5309            folded.push(label.to_string());
5310        }
5311        // The judges are finished; their checkouts are pure cost from here.
5312        let root = self.state.worktree_root();
5313        for j in 1..=self.roles.judges.len() {
5314            let wt = root.join(format!("judge-{j}"));
5315            if wt.exists() {
5316                remove_preserved_worktree(&repo, &wt).await?;
5317            }
5318        }
5319        // The design-deliberation stage is finished by the time a tally
5320        // exists — same reasoning as the judges above.
5321        if self.state.config.graph.advise {
5322            for k in 1..=self.state.config.graph.advisors {
5323                let wt = root.join(format!("advisor-{k}"));
5324                if wt.exists() {
5325                    remove_preserved_worktree(&repo, &wt).await?;
5326                }
5327            }
5328        }
5329        if !folded.is_empty() {
5330            self.state
5331                .event("fold", format!("folded candidates {}", folded.join(", ")));
5332            self.state.save()?;
5333        }
5334        Ok(())
5335    }
5336
5337    // ------------------------------------------------------------ base sync
5338
5339    async fn hold_base_sync(&mut self) -> Result<bool> {
5340        self.hold_base_sync_as(None).await
5341    }
5342
5343    async fn hold_base_sync_kind(
5344        &mut self,
5345        kind: crate::base_sync_recovery::StopKind,
5346    ) -> Result<()> {
5347        self.hold_base_sync_as(Some(kind)).await?;
5348        Ok(())
5349    }
5350
5351    async fn hold_base_sync_as(
5352        &mut self,
5353        kind: Option<crate::base_sync_recovery::StopKind>,
5354    ) -> Result<bool> {
5355        if !self.state.base_sync_blocked() {
5356            self.state.base_sync_recovery.stop = None;
5357            return Ok(false);
5358        }
5359        let head = match self.state.winner() {
5360            Some(w) => git::rev_parse(&w.worktree, "HEAD")
5361                .await
5362                .unwrap_or_default(),
5363            None => String::new(),
5364        };
5365        if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5366            && !authorization.consumed
5367            && authorization.stop.head != head
5368        {
5369            authorization.refuse(format!(
5370                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5371                authorization.stop.head
5372            ));
5373        }
5374        self.state.record_base_sync_stop(&head);
5375        if let Some(kind) = kind {
5376            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5377            if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5378                stop.kind = kind;
5379            }
5380        }
5381        self.state.save()?;
5382        Ok(true)
5383    }
5384
5385    /// Land the winner's tree on the current tip of `<remote>/<base>` before
5386    /// anything verifies it.
5387    ///
5388    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
5389    /// read whatever is checked out in the winner's worktree. Left alone that
5390    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
5391    /// it when the run *branched* - and a run takes long enough that the base
5392    /// has usually moved by the time it gets here. A gate that ran there
5393    /// answers "green on the commit this run started from", not "green on
5394    /// what is about to land", and the difference showed up three times in
5395    /// one day as a green run whose merge would have reverted a file another
5396    /// pull request had already landed.
5397    ///
5398    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5399    /// `land::Step::Rebase` calls, rather than a second implementation of the
5400    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5401    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5402    /// doc warns about ("two answers to a question nobody notices until a
5403    /// diff is wrong").
5404    ///
5405    /// Foreign-checkout reviews instead merge the base into the owned shadow
5406    /// worktree, preserving the ancestry required for fast-forward delivery.
5407    /// A conflicting merge is aborted and held for operator resolution.
5408    ///
5409    /// A conflict is not the end of the road: the standing rebase is handed
5410    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5411    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5412    /// land). Once it finishes, review and the gate run as usual on the
5413    /// rebased tree, which is where a breakage the new base caused is caught
5414    /// by the ordinary gate-fix round. magi resolves nothing itself.
5415    ///
5416    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5417    /// the current durable recovery interval, bounds how often the base is
5418    /// synchronized during active recovery (lifetime attempts remain evidence).
5419    /// `rebase_fixes` is how many times a *conflict* was given to a fixer. When the fixer cannot finish the
5420    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5421    /// what was tried (rounds spent, paths still conflicted) and the branch
5422    /// and worktree stay exactly as they were - untouched, for a person to
5423    /// look at - which is also what makes re-entering this function
5424    /// afterwards a no-op instead of a second attempt at the same wall. A
5425    /// push failure ends the same way.
5426    async fn sync_to_base(&mut self) -> Result<()> {
5427        if self.state.status == RunStatus::AlreadyInBase {
5428            return Ok(());
5429        }
5430        self.hold_base_sync().await?;
5431        let conflicted = self
5432            .state
5433            .base_sync
5434            .as_ref()
5435            .is_some_and(|s| s.conflict.is_some());
5436        let Some(winner) = self.state.winner().cloned() else {
5437            return Ok(());
5438        };
5439
5440        crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5441
5442        let repo = self.state.repo.clone();
5443        let remote = self.state.config.merge.remote.clone();
5444        let base_branch = self.state.base_branch.clone();
5445        let tracking = format!("{remote}/{base_branch}");
5446
5447        let fetched_base =
5448            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5449        // No network, or the remote never had this branch: the run already
5450        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5451        // otherwise), and one that got this far is not blocked by a fetch
5452        // that fails now. It just has no newer tip to compare against.
5453        let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5454            return Ok(());
5455        };
5456
5457        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5458        self.state.ensure_base_episode();
5459        let mut pending = self
5460            .state
5461            .base_sync_recovery
5462            .operations
5463            .last()
5464            .filter(|o| !o.finished)
5465            .cloned();
5466        let scratch = self.state.dir().join("base-sync");
5467        if git::rebase_in_progress(&scratch).await {
5468            // Older resource stops predate operation reservations. Import the
5469            // standing Git operation, never retarget it to today's main.
5470            if pending.is_none()
5471                && self.state.base_sync_recovery.operations.is_empty()
5472                && !conflicted
5473                && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5474                    .await
5475                    .ok()
5476                    .as_deref()
5477                    == Some(head.as_str())
5478                && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5479                && git::is_clean(&winner.worktree).await?
5480            {
5481                let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5482                if self.state.base_sync.is_none() {
5483                    self.state.base_sync = Some(BaseSync {
5484                        tip: saved_tip.clone(),
5485                        behind: 0,
5486                        attempts: 0,
5487                        conflict: None,
5488                        already_in: None,
5489                    });
5490                }
5491                self.state.reserve_base_operation(&head, &saved_tip);
5492                self.state.event(
5493                    "base-recovery",
5494                    "imported standing legacy rebase as a spent synchronization reservation",
5495                );
5496                self.state.save()?;
5497                pending = self.state.base_sync_recovery.operations.last().cloned();
5498            }
5499            let expected = pending.as_ref().map(|op| op.tip.as_str());
5500            if expected.is_none()
5501                || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5502            {
5503                let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5504                if self.state.base_sync.is_none() {
5505                    self.state.base_sync = Some(BaseSync {
5506                        tip: tip.clone(),
5507                        behind: 0,
5508                        attempts: 0,
5509                        conflict: None,
5510                        already_in: None,
5511                    });
5512                }
5513                self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5514                self.state.event("land", why);
5515                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5516                    .await?;
5517                return Ok(());
5518            }
5519        }
5520        if let Some(op) = &pending {
5521            // A replay may add the pinned base's commits, never foreign work.
5522            let mut replay_proven =
5523                head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5524            if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5525                let wait = repository_wait(&self.state, None);
5526                let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5527                match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5528                    Ok(true) => {
5529                        self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5530                        self.state.save()?;
5531                        git::worktree_remove(&repo, &scratch).await.ok();
5532                        replay_proven = true;
5533                    }
5534                    Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5535                    Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5536                }
5537            }
5538            if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5539                if self.state.foreign_checkout.is_some() {
5540                    replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5541                        .await
5542                        .ok()
5543                        .as_deref()
5544                        == Some(op.head.as_str())
5545                        && git::rev_parse(&repo, &format!("{head}^2"))
5546                            .await
5547                            .ok()
5548                            .as_deref()
5549                            == Some(op.tip.as_str());
5550                } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5551                    .await
5552                    .is_ok_and(|m| m.is_empty())
5553                    && let Ok(added) =
5554                        crate::reconcile::origin_missing(&repo, &op.head, &head).await
5555                {
5556                    replay_proven = true;
5557                    for commit in added {
5558                        if !git::is_ancestor(&repo, &commit, &op.tip).await {
5559                            replay_proven = false;
5560                            break;
5561                        }
5562                    }
5563                    replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5564                        <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5565                }
5566            }
5567            // Never retarget an operation across a crash, even if main moved.
5568            // sync_to_head resets and cleans with -x, so even ignored untracked
5569            // files must stop recovery independently of the old tracked tree.
5570            if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5571                || (head != op.head
5572                    && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5573                        .await?
5574                        .is_empty())
5575                || (!git::is_clean(&winner.worktree).await?
5576                    && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5577                        .await
5578                        .is_ok_and(|o| o.ok()))
5579                || !replay_proven
5580            {
5581                let why = format!(
5582                    "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5583                    op.id
5584                );
5585                let sync = self
5586                    .state
5587                    .base_sync
5588                    .as_mut()
5589                    .context("reserved synchronization has no lifetime accounting")?;
5590                sync.conflict = Some(why.clone());
5591                self.state.event("land", why);
5592                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5593                    .await?;
5594                return Ok(());
5595            }
5596            if head != op.head {
5597                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5598            }
5599            tip = op.tip.clone();
5600        }
5601        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5602        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5603        if pending.is_none()
5604            && !conflicted
5605            && fetched_base
5606            && self
5607                .state
5608                .base_sync_recovery
5609                .episodes
5610                .last()
5611                .is_some_and(|e| e.boundary.is_some())
5612            && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5613            && let Ok(owner) = crate::queue::Queue::open().get(&task)
5614            && owner.execution_runs().last() == Some(&self.state.id)
5615            && owner.status != crate::queue::TaskStatus::Done
5616            && !owner.fresh_start
5617            && owner.review_branch.is_none()
5618            && crate::daemon::compatible_run_instruction(&self.state, &owner)
5619            && let Some(approval) = owner.resumption.as_ref()
5620            && approval.instruction == owner.instruction
5621            && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5622            && git::is_clean(&winner.worktree).await?
5623            && self.state.renew_base_episode(&task, approval, &head, &tip)
5624        {
5625            self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5626            self.state.save()?;
5627        }
5628
5629        // Before any rebase, and before a recorded conflict is honoured: a
5630        // branch whose change reached the base under other commit ids has
5631        // nothing to rebase and nothing to conflict with, and a run that
5632        // already stopped on that phantom conflict recovers here on resume.
5633        // `behind == 0` with head == tip is a branch the base has since taken
5634        // in whole, whether or not a conflict was ever recorded: the ancestry
5635        // proof must still run (`classify` ignores a head still on the start
5636        // commit).
5637        if (behind > 0 || conflicted || head == tip)
5638            && self
5639                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5640                .await?
5641        {
5642            return Ok(());
5643        }
5644        if conflicted && self.state.base_recovery_pending() {
5645            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5646            let owner =
5647                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5648            let owner_valid = owner.as_ref().is_ok_and(|t| {
5649                t.execution_runs().last() == Some(&self.state.id)
5650                    && t.status != crate::queue::TaskStatus::Done
5651                    && !t.fresh_start
5652                    && t.review_branch.is_none()
5653            });
5654            let invalidation = if !owner_valid {
5655                Some("latest task ownership changed")
5656            } else if !owner.as_ref().is_ok_and(|t| {
5657                authorization
5658                    .generation
5659                    .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5660            }) {
5661                Some("owner approval generation changed")
5662            } else if authorization.question.as_ref().is_some_and(|question| {
5663                !crate::base_recovery_control::decision_current(
5664                    &crate::queue::Queue::at(crate::run::home().join("queue")),
5665                    &self.state,
5666                    &authorization.task,
5667                    question,
5668                )
5669            }) {
5670                Some("recovery decision ownership, routing or stop evidence changed")
5671            } else if authorization.request.is_some()
5672                && !owner
5673                    .as_ref()
5674                    .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5675            {
5676                Some("owner decision changed")
5677            } else if let Some(scope) = &authorization.scope {
5678                if scope.tip != tip {
5679                    Some("fetched base changed")
5680                } else if !self
5681                    .state
5682                    .base_sync_recovery
5683                    .episodes
5684                    .last()
5685                    .is_some_and(|e| e.id == scope.episode)
5686                {
5687                    Some("active synchronization interval changed")
5688                } else if !owner
5689                    .as_ref()
5690                    .is_ok_and(|t| t.instruction == scope.instruction)
5691                {
5692                    Some("task instruction changed")
5693                } else {
5694                    None
5695                }
5696            } else {
5697                None
5698            };
5699            if let Some(reason) = invalidation {
5700                let authorization = self
5701                    .state
5702                    .base_sync_recovery
5703                    .authorizations
5704                    .last_mut()
5705                    .unwrap();
5706                authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5707                self.state.save()?;
5708                return Ok(());
5709            }
5710            if !fetched_base
5711                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5712                || !git::is_clean(&winner.worktree).await?
5713                || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5714            {
5715                return Ok(());
5716            }
5717        }
5718        let recovery = conflicted && self.state.consume_base_recovery(&head);
5719        if conflicted {
5720            // Consumption (including a refused changed HEAD) is durable before Git.
5721            self.state.save()?;
5722            if !recovery {
5723                return Ok(());
5724            }
5725        }
5726
5727        if behind == 0 {
5728            // A fixer-finished rebase moves the branch ref before the
5729            // winner's worktree is told (`sync_to_head` below). A run that
5730            // died in between resumes here with `behind == 0` and a tree still
5731            // holding the pre-rebase files, which review and the gate would
5732            // then read. That state is exactly: HEAD moved off the tip the
5733            // rebase started from, yet the tree is still identical to that
5734            // tip. A tree with edits of its own differs from it, so nothing
5735            // is thrown away.
5736            if let Some(from) = self
5737                .state
5738                .rebase_fixes
5739                .iter()
5740                .rev()
5741                .find_map(|r| r.from.clone())
5742                && from != head
5743                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5744                    .await
5745                    .is_ok_and(|o| o.ok())
5746            {
5747                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5748            }
5749            // An earlier attempt may have rebased the branch locally and died
5750            // before pushing it (only the fresh-rebase arm below pushes).
5751            // Publish it now, so the plain push at PR time is not refused as
5752            // a non-fast-forward. A run already holding a recorded conflict
5753            // never reaches here; that case is out of scope.
5754            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5755            if let Some(why) = &conflict {
5756                self.state.status = RunStatus::Blocked;
5757                self.state.event("land", why.clone());
5758            }
5759            self.state.base_sync = Some(BaseSync {
5760                tip,
5761                behind: 0,
5762                attempts,
5763                conflict,
5764                already_in: None,
5765            });
5766            if pending.is_some() {
5767                self.state.finish_base_operation(Some(&head));
5768            }
5769            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5770                .await?;
5771            self.state.save()?;
5772            return Ok(());
5773        }
5774
5775        if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5776            let why = format!(
5777                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5778                 synchronization(s); synchronizing again would only race it",
5779                winner.branch
5780            );
5781            self.state.status = RunStatus::Blocked;
5782            self.state.base_sync = Some(BaseSync {
5783                tip,
5784                behind,
5785                attempts,
5786                conflict: Some(why.clone()),
5787                already_in: None,
5788            });
5789            self.state.event("land", why);
5790            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5791                .await?;
5792            self.state.save()?;
5793            return Ok(());
5794        }
5795
5796        if self.state.foreign_checkout.is_some() {
5797            // Delivery to the operator's checkout is fast-forward only. A
5798            // rebase here would discard its saved head from our ancestry.
5799            let wait = repository_wait(&self.state, None);
5800            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5801            if self.state.base_sync.is_none() {
5802                self.state.base_sync = Some(BaseSync {
5803                    tip: tip.clone(),
5804                    behind,
5805                    attempts: 0,
5806                    conflict: None,
5807                    already_in: None,
5808                });
5809            }
5810            self.state.reserve_base_operation(&head, &tip);
5811            self.state.save()?;
5812            let merged = git::git_raw(
5813                &winner.worktree,
5814                &[
5815                    "-c",
5816                    "core.hooksPath=/dev/null",
5817                    "merge",
5818                    "--no-ff",
5819                    "--no-autostash",
5820                    "--no-edit",
5821                    "-m",
5822                    &format!("Merge {tracking} before verifying"),
5823                    &tip,
5824                ],
5825            )
5826            .await?;
5827            let conflict = if merged.ok() {
5828                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5829                self.state.event(
5830                    "land",
5831                    format!("merged {tracking} into {} before verifying", winner.branch),
5832                );
5833                None
5834            } else {
5835                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5836                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5837                }
5838                let why = format!(
5839                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5840                    winner.branch,
5841                    merged.stderr.chars().take(600).collect::<String>()
5842                );
5843                self.state.status = RunStatus::Blocked;
5844                self.state.event("land", &why);
5845                Some(why)
5846            };
5847            let synced = conflict.is_none();
5848            let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5849            self.state
5850                .finish_base_operation(synced.then_some(fresh.as_str()));
5851            let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5852            self.state.base_sync = Some(BaseSync {
5853                tip,
5854                behind: if synced { 0 } else { behind },
5855                attempts,
5856                conflict,
5857                already_in: None,
5858            });
5859            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5860                .await?;
5861            self.state.save()?;
5862            return Ok(());
5863        }
5864
5865        self.state.event(
5866            "land",
5867            format!(
5868                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5869                winner.branch
5870            ),
5871        );
5872        self.state.save()?;
5873
5874        // The remote's copy of the branch, read now and only if the fetch
5875        // really succeeded (a stale tracking ref must never pin a lease). It is
5876        // pushed over after a rebase only when it is a commit this branch
5877        // already contains, by ancestry or by patch (an earlier rebase of ours
5878        // that never reached the remote): anything else is somebody else's work.
5879        let branch_tracking = format!("{remote}/{}", winner.branch);
5880        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5881        let remote_tip = if self.state.task_pr_branch().is_none()
5882            && matches!(&fetched_branch, Ok(o) if o.ok())
5883        {
5884            git::rev_parse(&repo, &branch_tracking).await.ok()
5885        } else {
5886            None
5887        };
5888        // A remote tip this branch does not contain is somebody else's work:
5889        // rebasing would leave a local tip that can never be pushed. Stop
5890        // before touching anything and say so.
5891        if let Some(theirs) = &remote_tip
5892            && !git::is_ancestor(&repo, theirs, &head).await
5893            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5894                .await
5895                .is_ok_and(|missing| missing.is_empty())
5896        {
5897            let why = format!(
5898                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5899                 them",
5900                short(theirs),
5901                winner.branch
5902            );
5903            self.state.status = RunStatus::Blocked;
5904            self.state.base_sync = Some(BaseSync {
5905                tip,
5906                behind,
5907                attempts,
5908                conflict: Some(why.clone()),
5909                already_in: None,
5910            });
5911            self.state.event("land", why);
5912            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5913                .await?;
5914            self.state.save()?;
5915            return Ok(());
5916        }
5917
5918        if self.state.base_sync.is_none() {
5919            self.state.base_sync = Some(BaseSync {
5920                tip: tip.clone(),
5921                behind,
5922                attempts: 0,
5923                conflict: None,
5924                already_in: None,
5925            });
5926        }
5927        self.state.reserve_base_operation(&head, &tip);
5928        self.state.save()?;
5929        let scratch = self.state.dir().join("base-sync");
5930        let rebased =
5931            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5932                .await
5933            {
5934                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5935                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5936                Err(e) => Err(e),
5937            };
5938        let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5939        let stop_kind = match &rebased {
5940            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5941            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5942            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5943        };
5944        match rebased {
5945            Ok(None) => {
5946                // The branch ref moved, but a worktree that already had it
5947                // checked out (the winner's) was not told; sync its index and
5948                // files before anything reads them.
5949                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5950                let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5951                self.state.finish_base_operation(Some(&fresh));
5952                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5953                let mut conflict = None;
5954                if let Some(pinned) = &remote_tip {
5955                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5956                    match pushed {
5957                        Ok(o) if o.ok() => self.state.event(
5958                            "land",
5959                            format!("pushed rebased {} to {remote}", winner.branch),
5960                        ),
5961                        Ok(o) => {
5962                            conflict = Some(format!(
5963                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5964                                winner.branch,
5965                                short(pinned),
5966                                o.stderr.chars().take(600).collect::<String>()
5967                            ));
5968                        }
5969                        Err(e) => {
5970                            conflict = Some(format!(
5971                                "rebased {} locally but could not push it: {e:#}",
5972                                winner.branch
5973                            ));
5974                        }
5975                    }
5976                }
5977                if let Some(why) = &conflict {
5978                    self.state.status = RunStatus::Blocked;
5979                    self.state.event("land", why.clone());
5980                }
5981                self.state.base_sync = Some(BaseSync {
5982                    tip: tip.clone(),
5983                    behind: 0,
5984                    attempts,
5985                    conflict,
5986                    already_in: None,
5987                });
5988                self.state
5989                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
5990            }
5991            Ok(Some(conflict)) => {
5992                self.state.finish_base_operation(None);
5993                let why = format!(
5994                    "{} conflicts with {tracking} and did not rebase: {}",
5995                    winner.branch,
5996                    conflict.chars().take(600).collect::<String>()
5997                );
5998                self.state.status = RunStatus::Blocked;
5999                self.state.base_sync = Some(BaseSync {
6000                    tip,
6001                    behind,
6002                    attempts,
6003                    conflict: Some(why.clone()),
6004                    already_in: None,
6005                });
6006                self.state.event("land", why);
6007            }
6008            // A refused Git write or a busy repository: nothing is wrong with
6009            // the branch. The scratch rebase stays standing and the run
6010            // resumes into it (`base_sync` must not claim a conflict).
6011            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6012                self.state.save()?;
6013                return Ok(());
6014            }
6015            Err(e) => {
6016                self.state.finish_base_operation(None);
6017                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6018                self.state.status = RunStatus::Blocked;
6019                self.state.base_sync = Some(BaseSync {
6020                    tip,
6021                    behind,
6022                    attempts,
6023                    conflict: Some(why.clone()),
6024                    already_in: None,
6025                });
6026                self.state.event("land", why);
6027            }
6028        }
6029        if !self.state.base_sync_blocked() {
6030            self.state.base_sync_recovery.stop = None;
6031        }
6032        self.hold_base_sync_kind(stop_kind).await?;
6033        self.state.save()?;
6034        Ok(())
6035    }
6036
6037    /// Push a branch an earlier attempt rebased locally but never published,
6038    /// pinned to the remote tip read right after a successful fetch. Returns
6039    /// the reason when the run must stop; `None` when there was nothing to do
6040    /// (no remote copy, the same tip, or a remote copy this branch already
6041    /// contains, which the PR-time push fast-forwards) or the push succeeded.
6042    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6043        // Task competitions publish only after the PR text gate, to their
6044        // public head; never resurrect a candidate branch on the remote.
6045        if self.state.task_pr_branch().is_some() {
6046            return None;
6047        }
6048        let repo = self.state.repo.clone();
6049        let remote = self.state.config.merge.remote.clone();
6050        let fetched = git::fetch(&repo, &remote, branch).await;
6051        if !matches!(&fetched, Ok(o) if o.ok()) {
6052            return None;
6053        }
6054        let branch_tracking = format!("{remote}/{branch}");
6055        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6056        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6057            return None;
6058        }
6059        if !crate::reconcile::origin_missing(&repo, head, &theirs)
6060            .await
6061            .is_ok_and(|missing| missing.is_empty())
6062        {
6063            return Some(format!(
6064                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6065                 them",
6066                short(&theirs)
6067            ));
6068        }
6069        match git::push_pinned(&repo, &remote, branch, &theirs).await {
6070            Ok(o) if o.ok() => {
6071                self.state
6072                    .event("land", format!("pushed rebased {branch} to {remote}"));
6073                None
6074            }
6075            Ok(o) => Some(format!(
6076                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6077                 someone may have pushed): {}",
6078                short(&theirs),
6079                o.stderr.chars().take(600).collect::<String>()
6080            )),
6081            Err(e) => Some(format!(
6082                "{branch} is rebased locally but could not be pushed: {e:#}"
6083            )),
6084        }
6085    }
6086
6087    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
6088    /// is already on `tip` under other commit ids ([`crate::already`]); returns
6089    /// whether it did.
6090    ///
6091    /// Checked only when the base is ahead of the branch. A failing check is
6092    /// "not proven" - the ordinary rebase path then decides - never a reason to
6093    /// stop the run.
6094    ///
6095    /// The remote copy of the branch is held to the same standard as the local
6096    /// one: if it carries a tip this worktree does not, that tip must itself be
6097    /// proven in the base, or nothing is settled (a pull request would
6098    /// otherwise be closed over commits nobody checked). The pull request is
6099    /// closed *before* the terminal status is saved; if that fails for a
6100    /// reason other than a refusal (no network, a `gh` error) the run is left
6101    /// `Blocked` with the reason as its conflict, which a resume retries -
6102    /// the same recovery a phantom conflict gets.
6103    async fn settle_already_in(
6104        &mut self,
6105        branch: &str,
6106        tip: &str,
6107        head: &str,
6108        attempts: usize,
6109        behind: usize,
6110    ) -> Result<bool> {
6111        let repo = self.state.repo.clone();
6112        let remote = self.state.config.merge.remote.clone();
6113        let start = self.state.base_commit.clone();
6114        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6115            Ok(Some(e)) => e,
6116            Ok(None) => return Ok(false),
6117            Err(e) => {
6118                tracing::warn!("already-in-base check for {branch}: {e:#}");
6119                return Ok(false);
6120            }
6121        };
6122        let mut verified = vec![head.to_owned()];
6123        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6124        let fetched = git::fetch(&repo, &remote, &public_branch).await;
6125        if matches!(&fetched, Ok(o) if o.ok())
6126            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6127            && theirs != head
6128        {
6129            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6130                Ok(Some(_)) => verified.push(theirs),
6131                _ => return Ok(false),
6132            }
6133        }
6134        let base_branch = self.state.base_branch.clone();
6135        let message = format!(
6136            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6137             land",
6138            evidence.names(),
6139            evidence.proof.as_str()
6140        );
6141        let closed =
6142            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6143                .await;
6144        match closed {
6145            Ok(Ok(url)) => self
6146                .state
6147                .event("land", format!("closed {url}: superseded on {base_branch}")),
6148            Ok(Err(why)) => self
6149                .state
6150                .event("land", format!("did not close a pull request: {why}")),
6151            Err(e) => {
6152                let why = format!(
6153                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
6154                     not be closed ({e:#}); resume to retry"
6155                );
6156                self.state.status = RunStatus::Blocked;
6157                self.state.base_sync = Some(BaseSync {
6158                    tip: tip.to_owned(),
6159                    behind,
6160                    attempts,
6161                    conflict: Some(why.clone()),
6162                    already_in: None,
6163                });
6164                self.state.event("land", why);
6165                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6166                    .await?;
6167                self.state.save()?;
6168                return Ok(true);
6169            }
6170        }
6171        self.state.status = RunStatus::AlreadyInBase;
6172        self.state.base_sync_recovery.stop = None;
6173        self.state.base_sync = Some(BaseSync {
6174            tip: tip.to_owned(),
6175            behind,
6176            attempts,
6177            conflict: None,
6178            already_in: Some(evidence),
6179        });
6180        self.state.event("land", message);
6181        self.state.save()?;
6182        self.settle_questions();
6183        Ok(true)
6184    }
6185
6186    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
6187    /// last landed the winner on, once it has run, else the commit the run
6188    /// branched from.
6189    ///
6190    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
6191    /// and `vote` all happen before there is a winner to rebase, so they
6192    /// compare every candidate against the branch point on purpose, and a
6193    /// base that moves after they are already done cannot change an answer
6194    /// they already gave.
6195    fn landing_base(&self) -> String {
6196        self.state
6197            .base_sync
6198            .as_ref()
6199            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6200    }
6201
6202    // ------------------------------------------------------- operator fix
6203
6204    /// Route specific, already-recorded review findings to a fixer for a
6205    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
6206    /// entry point.
6207    ///
6208    /// Distinct from `review_loop`'s own fix step in three ways: it never
6209    /// runs a reviewer wave, it never spends review-round budget, and what
6210    /// happened is recorded as an [`OperatorFixRequest`] appended to
6211    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
6212    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
6213    /// and vote must never be rewritten to look like a manufactured blocking
6214    /// verdict.
6215    ///
6216    /// Only meaningful once review has actually concluded: `Ready` (handed
6217    /// off with findings still open, or simply concluded clean while minor
6218    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
6219    /// gate failed). Everything else is refused: a run still in progress
6220    /// should simply be resumed, and a `Merged` run's branch has already
6221    /// landed — reopening *this* run's own record cannot change that, so the
6222    /// answer there is a fresh `magi review <branch>`.
6223    ///
6224    /// A real commit here re-verifies through a fresh, ordinary review-only
6225    /// run on the same branch ([`Self::review`]) rather than reopening this
6226    /// run's own `review_loop`: once any round in this run's history went
6227    /// clean, `review_conclusion` treats that as permanent by design (the
6228    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
6229    /// way to force one more genuine reviewer wave out of *this* run without
6230    /// either rewriting history or weakening that guarantee for every other
6231    /// caller. A review-only run costs nothing extra — no implementation, no
6232    /// judging, no vote — and exercises the exact same review → verify →
6233    /// gate → (human) merge path, unmodified.
6234    pub async fn fix_selected(
6235        &mut self,
6236        ids: &[String],
6237        reason: &str,
6238        allow_stale: bool,
6239    ) -> Result<()> {
6240        let reason = reason.trim();
6241        if reason.is_empty() {
6242            bail!("a fix request needs a reason — that is the operator's own record of why");
6243        }
6244        if ids.is_empty() {
6245            bail!("no finding id given");
6246        }
6247        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6248            bail!(
6249                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6250                 has already concluded — can be given a targeted fix. A run still \
6251                 in progress should simply be resumed; a `merged` run's branch has \
6252                 already landed, so its answer is a fresh `magi review <branch>`, \
6253                 not reopening this run's own record",
6254                self.state.id,
6255                self.state.status.as_str()
6256            );
6257        }
6258        // Reassignment shares this claim through its atomic task save. Check
6259        // the canonical run identity inside it, before preflight or any await.
6260        let _claim = FixClaim::acquire(&self.state.dir())?;
6261        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6262        let Some(winner) = self.state.winner().cloned() else {
6263            bail!("run {} has no winning candidate to fix", self.state.id);
6264        };
6265        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6266            bail!(
6267                "branch `{}` no longer exists; this run cannot be extended",
6268                winner.branch
6269            );
6270        }
6271        let home = crate::run::home();
6272        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6273            bail!(
6274                "run {} is currently being worked on by another magi process",
6275                self.state.id
6276            );
6277        }
6278        // Resolve every id before spending anything — an unknown id refuses
6279        // the whole request rather than silently dropping it — and dedup
6280        // while keeping the operator's own order.
6281        let mut seen = BTreeSet::new();
6282        let mut findings = Vec::new();
6283        let mut missing = Vec::new();
6284        for id in ids {
6285            if !seen.insert(id.clone()) {
6286                continue;
6287            }
6288            match self.state.finding(id) {
6289                Some((round, rec, f)) => findings.push(OperatorFixFinding {
6290                    id: f.id.clone(),
6291                    severity: f.severity,
6292                    reviewer_vote: rec.vote,
6293                    round: round.round,
6294                    round_head: round.head.clone(),
6295                    reviewer: rec.reviewer,
6296                    agent: rec.agent.clone(),
6297                    file: f.file.clone(),
6298                    line: f.line,
6299                    title: f.title.clone(),
6300                    detail: f.detail.clone(),
6301                    outcome: OperatorFixOutcome::Pending,
6302                }),
6303                None => missing.push(id.clone()),
6304            }
6305        }
6306        if !missing.is_empty() {
6307            bail!(
6308                "unknown finding id(s): {}; nothing was changed",
6309                missing.join(", ")
6310            );
6311        }
6312
6313        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6314        let stale_details: Vec<(String, String)> = findings
6315            .iter()
6316            .filter(|f| f.round_head != head_at_request)
6317            .map(|f| (f.id.clone(), f.round_head.clone()))
6318            .collect();
6319        let stale = !stale_details.is_empty();
6320        if stale && !allow_stale {
6321            bail!(
6322                "the branch has moved since some finding(s) were raised — {} — now \
6323                 at {}; pass --allow-stale to fix anyway, or re-run review first",
6324                stale_details
6325                    .iter()
6326                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6327                    .collect::<Vec<_>>()
6328                    .join(", "),
6329                short(&head_at_request)
6330            );
6331        }
6332
6333        let request = OperatorFixRequest {
6334            requested_at: Timestamp::now(),
6335            reason: reason.to_owned(),
6336            findings,
6337            head_at_request: head_at_request.clone(),
6338            allow_stale,
6339            stale,
6340            fix: None,
6341            result_head: None,
6342            follow_up_review_run: None,
6343        };
6344        self.state.event(
6345            "fix",
6346            format!(
6347                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6348                request.findings.len(),
6349                request
6350                    .findings
6351                    .iter()
6352                    .map(|f| f.id.as_str())
6353                    .collect::<Vec<_>>()
6354                    .join(", "),
6355            ),
6356        );
6357        // Recorded now, before any worktree work or the fixer call itself —
6358        // and re-saved at each checkpoint below: a crash at any point after
6359        // this (mid fixer call, mid follow-up review) must not lose the fact
6360        // that this was requested, for which findings, and why. Everything
6361        // past this point reads and writes through `request_index` rather
6362        // than a local variable, since `request` itself is moved here.
6363        self.state.operator_fixes.push(request);
6364        self.state.save()?;
6365        let request_index = self.state.operator_fixes.len() - 1;
6366
6367        // A fresh, dedicated worktree for this one call, never the winner's
6368        // own worktree in place: that one may already be gone (folded away),
6369        // and reusing it in place would leave the branch checked out there
6370        // when the follow-up review below tries to check it out again. Freed
6371        // immediately after, either way — but only once confirmed clean:
6372        // `worktree_remove` is a `git worktree remove --force`, which would
6373        // otherwise discard uncommitted work left there by the operator or
6374        // another process before this had a chance to even look at it.
6375        let repo = self.state.repo.clone();
6376        let coordination = self.coordinate(&repo).await?;
6377        if winner.worktree.exists() {
6378            // Lockfiles a rescue commit withheld stay untracked on purpose and
6379            // are already recorded; they are not the operator's work to protect.
6380            let dirty = git::git(
6381                &winner.worktree,
6382                &["status", "--porcelain", "--untracked-files=all"],
6383            )
6384            .await?;
6385            let only_withheld = dirty.lines().all(|l| {
6386                l.strip_prefix("?? ")
6387                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6388            });
6389            if !only_withheld {
6390                bail!(
6391                    "`{}` has uncommitted changes; refusing to touch it — commit or \
6392                     discard them first",
6393                    winner.worktree.display()
6394                );
6395            }
6396            git::worktree_remove(&self.state.repo, &winner.worktree)
6397                .await
6398                .ok();
6399        }
6400        let fix_worktree = self.state.worktree_root().join("operator-fix");
6401        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6402        git::git(
6403            &self.state.repo,
6404            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6405        )
6406        .await
6407        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6408        if !git::is_clean(&fix_worktree).await? {
6409            git::worktree_remove(&self.state.repo, &fix_worktree)
6410                .await
6411                .ok();
6412            bail!(
6413                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6414                winner.branch
6415            );
6416        }
6417        let repo_for_setup = self.state.repo.clone();
6418        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6419            Ok(()) => {
6420                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6421            }
6422            Err(e) => Err(e),
6423        };
6424        if let Err(e) = setup {
6425            git::worktree_remove(&self.state.repo, &fix_worktree)
6426                .await
6427                .ok();
6428            return Err(e);
6429        }
6430
6431        // This explicit operator request retries availability independently
6432        // of execute(); the previous call has already saved and cleaned up.
6433        self.state.resource_block = None;
6434
6435        let run_id = self.state.id.clone();
6436        let prompts = self.state.config.prompts.clone();
6437        let language = self.state.config.graph.language.clone();
6438        let sessions = self.state.config.graph.sessions;
6439        let artifacts = agent::artifacts_dir(&self.state.dir());
6440        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6441            .findings
6442            .iter()
6443            .map(|f| Finding {
6444                id: f.id.clone(),
6445                severity: f.severity,
6446                file: f.file.clone(),
6447                line: f.line,
6448                title: f.title.clone(),
6449                detail: f.detail.clone(),
6450            })
6451            .collect();
6452        let fix_prompt = prompt::operator_fix_in(
6453            &self.state.instruction,
6454            &finding_list,
6455            reason,
6456            &stale_details,
6457            &head_at_request,
6458            &language,
6459            &self.state.config.graph.github_language,
6460        );
6461        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6462        drop(coordination);
6463        let (job, seat, out) = self
6464            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6465                prompt: fix_prompt.clone(),
6466                spec,
6467                seat,
6468                cwd: fix_worktree.clone(),
6469                timeout,
6470                allow_write: true,
6471                sessions,
6472                artifacts: artifacts.clone(),
6473                stem: "operator-fix".to_owned(),
6474                handover: None,
6475            })
6476            .await;
6477        let agent_id = seat.agent.clone();
6478
6479        let mut fix = FixRecord {
6480            agent: agent_id,
6481            addressed: Vec::new(),
6482            rejected: Vec::new(),
6483            notes: String::new(),
6484            committed: false,
6485            failed: None,
6486            duration_ms: 0,
6487            continuation: None,
6488        };
6489        let mut final_seat = seat.clone();
6490        match out {
6491            AgentOutcome::Ok(o) => {
6492                fix.duration_ms = o.duration_ms;
6493                let parsed = verdict::extract_json::<FixReport>(&o.text);
6494                let incomplete_reason = match &parsed {
6495                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6496                        "the reply parsed, but it reported a command whose own CLI \
6497                         never confirmed an exit status"
6498                            .to_owned(),
6499                    ),
6500                    Ok(_) => None,
6501                    Err(e) => Some(e.to_string()),
6502                };
6503                match incomplete_reason {
6504                    None => {
6505                        let report = parsed.expect("checked Ok above");
6506                        fix.addressed = report.addressed;
6507                        fix.rejected = report.rejected;
6508                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6509                    }
6510                    Some(reason) => {
6511                        let (resumed_seat, resolved, failure, cont) = self
6512                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6513                            .await;
6514                        fix.duration_ms += cont.cumulative_wait_ms;
6515                        fix.continuation = Some(cont);
6516                        final_seat = resumed_seat;
6517                        match resolved {
6518                            Some(report) => {
6519                                fix.addressed = report.addressed;
6520                                fix.rejected = report.rejected;
6521                                fix.notes =
6522                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6523                            }
6524                            None => fix.failed = failure,
6525                        }
6526                    }
6527                }
6528            }
6529            AgentOutcome::Dropped(o) => {
6530                fix.duration_ms = o.duration_ms;
6531                let why = o
6532                    .dropped
6533                    .as_ref()
6534                    .map(|d| d.why.as_str())
6535                    .unwrap_or("the CLI ended the stream without delivering its answer");
6536                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6537            }
6538            AgentOutcome::Quota(o) => {
6539                self.state.quota.push(QuotaLoss::from_report(
6540                    final_seat.key.clone(),
6541                    "fix",
6542                    o.quota.as_ref(),
6543                ));
6544                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6545            }
6546            AgentOutcome::Failed(e) => fix.failed = Some(e),
6547            AgentOutcome::ResourceBlocked(e) => {
6548                fix.failed = Some(format!("the fixer was not started: {e}"));
6549            }
6550        }
6551        // Even a deferred call must rescue edits, release the worktree and
6552        // finalize its record below so an explicit `magi fix` can retry.
6553        if fix.continuation.is_none() {
6554            fix.continuation = Some(ContinuationRecord::not_needed());
6555        }
6556        self.state.seats.insert(final_seat.key.clone(), final_seat);
6557
6558        let rescue_message = format!(
6559            "magi: operator-selected fix ({}) (uncommitted work)",
6560            self.state.operator_fixes[request_index]
6561                .findings
6562                .iter()
6563                .map(|f| f.id.as_str())
6564                .collect::<Vec<_>>()
6565                .join(", ")
6566        );
6567        let repo = self.state.repo.clone();
6568        let coordination = self.coordinate(&repo).await?;
6569        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6570            self.state.note_withheld("fix", &r.withheld);
6571        }
6572        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6573        fix.committed = after != head_at_request;
6574        git::worktree_remove(&self.state.repo, &fix_worktree)
6575            .await
6576            .ok();
6577        drop(coordination);
6578
6579        self.state.event(
6580            "fix",
6581            match &fix.failed {
6582                Some(reason) => format!(
6583                    "operator fix: adoption report was lost ({reason}); {}",
6584                    if fix.committed {
6585                        "committed"
6586                    } else {
6587                        "NO new commit"
6588                    }
6589                ),
6590                None => format!(
6591                    "operator fix: {} addressed, {} rejected, {}",
6592                    fix.addressed.len(),
6593                    fix.rejected.len(),
6594                    if fix.committed {
6595                        "committed"
6596                    } else {
6597                        "NO new commit"
6598                    }
6599                ),
6600            },
6601        );
6602
6603        // Every selected finding gets an outcome — never left `Pending` once
6604        // the fixer's own turn is over. A report that never came back at all
6605        // marks every one of them `Unreported`, not silently "not addressed":
6606        // quota, a dropped stream, or an exhausted continuation are gaps in
6607        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
6608        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
6609        for f in &mut self.state.operator_fixes[request_index].findings {
6610            f.outcome = if fix.failed.is_some() {
6611                OperatorFixOutcome::Unreported
6612            } else if fix.addressed.contains(&f.id) {
6613                OperatorFixOutcome::Addressed
6614            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6615                OperatorFixOutcome::Rejected { why: r.why.clone() }
6616            } else {
6617                OperatorFixOutcome::Unreported
6618            };
6619        }
6620
6621        let committed = fix.committed;
6622        if committed {
6623            self.state.operator_fixes[request_index].result_head = Some(after.clone());
6624        }
6625        self.state.operator_fixes[request_index].fix = Some(fix);
6626        // Saved again now that the fixer's own outcome is final, on top of
6627        // the save right after the request was first pushed above.
6628        self.state.save()?;
6629        if self.state.resource_block.is_some() {
6630            return self.stop_for_resource();
6631        }
6632
6633        if committed {
6634            self.state.event(
6635                "fix",
6636                format!(
6637                    "operator fix committed {}; opening a follow-up review-only run",
6638                    short(&after)
6639                ),
6640            );
6641            // The operator asked for the fix, and the follow-up serves whatever
6642            // task the run it follows served.
6643            let origin =
6644                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6645            match Self::review(
6646                &self.state.repo,
6647                &winner.branch,
6648                self.state.config.clone(),
6649                origin,
6650            )
6651            .await
6652            {
6653                Ok(mut follow_up) => {
6654                    follow_up.state.event(
6655                        "start",
6656                        format!(
6657                            "requested by an operator fix on run {} for finding(s) {}",
6658                            self.state.id,
6659                            self.state.operator_fixes[request_index]
6660                                .findings
6661                                .iter()
6662                                .map(|f| f.id.as_str())
6663                                .collect::<Vec<_>>()
6664                                .join(", "),
6665                        ),
6666                    );
6667                    follow_up.state.save()?;
6668                    let follow_up_id = follow_up.state.id.clone();
6669                    // The follow-up is a run like any other: it belongs to the
6670                    // task of the run it follows, or to one filed for it.
6671                    let adopted = match crate::direct::adopt(
6672                        &follow_up.state,
6673                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6674                    ) {
6675                        Ok(a) => a,
6676                        Err(e) => {
6677                            // An ownerless run must not spend agent calls; it
6678                            // stays saved, and `magi run --resume` adopts it.
6679                            self.state.event(
6680                                "fix",
6681                                format!(
6682                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6683                                ),
6684                            );
6685                            self.state.save()?;
6686                            return Ok(());
6687                        }
6688                    };
6689                    let executed = follow_up.execute().await;
6690                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6691                    if let Some(a) = adopted {
6692                        a.finish(&follow_up.state, executed);
6693                    }
6694                    if let Some(e) = failure {
6695                        self.state.event(
6696                            "fix",
6697                            format!(
6698                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6699                            ),
6700                        );
6701                    }
6702                    self.state.operator_fixes[request_index].follow_up_review_run =
6703                        Some(follow_up_id);
6704                }
6705                Err(e) => {
6706                    self.state.event(
6707                        "fix",
6708                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6709                    );
6710                }
6711            }
6712            self.state.save()?;
6713        }
6714
6715        Ok(())
6716    }
6717
6718    // --------------------------------------------------------------- review
6719
6720    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6721    /// names, in order, each at most once, moving on only when the call
6722    /// advances (an error, a quota hit or nothing usable - the same decision
6723    /// point `agent::chain_advances` is for the other chained roles).
6724    ///
6725    /// Each agent gets its own seat from `Runner::seat`: the same agent
6726    /// continues its conversation (the winner's own implementer seat when it
6727    /// is the winner's author, now that the competition is over), another
6728    /// takes a fresh one so the full prompt is sent again. A handover is
6729    /// recorded under `node`, which is what makes the fallback stick for the
6730    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6731    /// is returned, so an exhausted chain reads exactly like a single failed
6732    /// fixer: one quota loss, the same wording, the same refund. An earlier
6733    /// attempt's edits are left in the tree and judged, with the final
6734    /// attempt's, by what git says afterwards.
6735    ///
6736    /// Returns the job that produced the outcome, for `continue_fix_report`.
6737    async fn ask_fixer(
6738        &mut self,
6739        winner: &Candidate,
6740        node: &'static str,
6741        round: Option<usize>,
6742        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6743    ) -> (SeatJob, SeatState, AgentOutcome) {
6744        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6745        self.ask_fixer_among(attempts, node, round, None, build)
6746            .await
6747    }
6748
6749    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6750    /// worktree and a commit: before the next agent is asked after one that
6751    /// did not satisfy the call, the worktree is put back on that commit, so
6752    /// a failed agent's partial edits are not carried into the next one.
6753    async fn ask_fixer_among(
6754        &mut self,
6755        attempts: Vec<(AgentSpec, String)>,
6756        node: &'static str,
6757        round: Option<usize>,
6758        restore: Option<(&Path, &str)>,
6759        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6760    ) -> (SeatJob, SeatState, AgentOutcome) {
6761        let run_id = self.state.id.clone();
6762        let prompts = self.state.config.prompts.clone();
6763        let cache = self.state.config.cache_dir();
6764        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6765        let mut last = None;
6766        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6767            let seat = self.seat(&key, &spec.id);
6768            let mut job = build(spec.clone(), seat);
6769            if i > 0 {
6770                job.stem = format!("{}-{}", job.stem, spec.id);
6771            }
6772            let ctx = WaveCtx {
6773                carry_seats: false,
6774                pauses: Some((&self.pause, &self.interrupt)),
6775                run: &run_id,
6776                node,
6777                prompts: &prompts,
6778                cache: cache.as_deref(),
6779                round,
6780            };
6781            let (seat, out) =
6782                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6783            // A resource block never advances the chain: the chosen agent
6784            // was not asked, so it must be the one asked again on resume.
6785            let advances = match &out {
6786                AgentOutcome::Ok(o) => agent::output_advances(o),
6787                AgentOutcome::ResourceBlocked(_) => false,
6788                _ => true,
6789            };
6790            if let Some(next) = ids.get(i + 1)
6791                && advances
6792            {
6793                self.state.seats.insert(seat.key.clone(), seat.clone());
6794                let class =
6795                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6796                let reason = match &out {
6797                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6798                    other => fail_reason(other),
6799                };
6800                // A failed agent's edits must not reach the next one, and a
6801                // restore that did not work ends the chain: the call fails
6802                // and the caller's own (confirmed) restore decides.
6803                if let Some((wt, rev)) = restore
6804                    && let Err(e) = self.graft_restore(wt, rev).await
6805                {
6806                    let why =
6807                        format!("could not restore the worktree before the next fixer: {e:#}");
6808                    tracing::warn!("{why}");
6809                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6810                        AgentOutcome::ResourceBlocked(why)
6811                    } else {
6812                        AgentOutcome::Failed(why)
6813                    };
6814                    last = Some((job, seat, out));
6815                    break;
6816                }
6817                record_handover(
6818                    &mut self.state,
6819                    node,
6820                    &seat.key,
6821                    &spec.id,
6822                    next,
6823                    &class,
6824                    &reason,
6825                );
6826                continue;
6827            }
6828            last = Some((job, seat, out));
6829            break;
6830        }
6831        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6832        if let AgentOutcome::Quota(output) = &out
6833            && node != "graft"
6834        {
6835            // Graft is best effort: its caller records the loss and restores
6836            // the pre-graft tree, then proceeds to review and gate.
6837            // Every fallback was tried. Machine outages cannot be repaired by
6838            // spending a fix round; use each caller's resumable resource path.
6839            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6840            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6841            self.state.quota.push(loss);
6842            self.state.seats.insert(seat.key.clone(), seat.clone());
6843            self.state.status = RunStatus::Blocked;
6844            return (
6845                job,
6846                seat,
6847                AgentOutcome::ResourceBlocked(
6848                    "fixer unavailable; restore agent availability and resume".into(),
6849                ),
6850            );
6851        }
6852        (job, seat, out)
6853    }
6854
6855    /// The one repair a verification-only failure earns (see
6856    /// [`crate::recovery`]), before the first review round of a run that took
6857    /// a branch over.
6858    ///
6859    /// Saved as started *before* the fixer is called, so a resume never
6860    /// repairs twice; a start found on re-entry is rolled back to the commit
6861    /// it began from. A failed or unusable fixer leaves the tree as it was and
6862    /// the review goes on over it, so the first round's own verification
6863    /// reports the failure again. The failure is never carried into a round as
6864    /// a result: verification is re-run on whatever commit this leaves.
6865    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6866        use crate::recovery::{Mode, Repair};
6867        if !self.state.reviews.is_empty() {
6868            return Ok(());
6869        }
6870        let Some(mut rec) = self.state.recovery.clone() else {
6871            return Ok(());
6872        };
6873        if rec.mode != Mode::VerifyRepair {
6874            return Ok(());
6875        }
6876        let wt = winner.worktree.clone();
6877        let mut head = git::rev_parse(&wt, "HEAD").await?;
6878        match rec.repair.clone() {
6879            Repair::Done { .. } => return Ok(()),
6880            Repair::WaitingForRestore { before } => {
6881                self.graft_restore(&wt, &before).await?;
6882                head = before;
6883                rec.repair = Repair::NotStarted;
6884                self.state.recovery = Some(rec.clone());
6885                self.state.save()?;
6886            }
6887            Repair::Started { before } => {
6888                // Interrupted: the unfinished edits are not trusted.
6889                // An unconfirmed rollback blocks the run with `Started` kept,
6890                // so a resume retries it; the tree is never reviewed as is.
6891                if let Err(e) = self.graft_restore(&wt, &before).await {
6892                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6893                        return Err(e);
6894                    }
6895                    return self
6896                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6897                        .await;
6898                }
6899                rec.repair = Repair::Done {
6900                    before: before.clone(),
6901                    after: before,
6902                    committed: false,
6903                    failed: Some("interrupted; rolled back".to_owned()),
6904                };
6905                self.state
6906                    .event("recovery", "the repair was interrupted and rolled back");
6907                self.state.recovery = Some(rec);
6908                self.state.save()?;
6909                return Ok(());
6910            }
6911            Repair::NotStarted => {}
6912        }
6913        if crate::recovery::recheck(&mut rec, &head) {
6914            self.state.event("recovery", rec.summary());
6915            self.state.recovery = Some(rec);
6916            self.state.save()?;
6917            return Ok(());
6918        }
6919        rec.repair = Repair::Started {
6920            before: head.clone(),
6921        };
6922        self.state.recovery = Some(rec.clone());
6923        self.state.save()?;
6924
6925        let language = self.state.config.graph.language.clone();
6926        let sessions = self.state.config.graph.sessions;
6927        let artifacts = agent::artifacts_dir(&self.state.dir());
6928        let fix_prompt = prompt::recovery_fix(
6929            &self.state.instruction,
6930            &rec,
6931            self.state.config.graph.review_rounds,
6932            &language,
6933        );
6934        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6935        self.state.event(
6936            "recovery",
6937            format!(
6938                "repair: the fixer starts from the failed verification on {}",
6939                short(&head)
6940            ),
6941        );
6942        let (_job, seat, out) = self
6943            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6944                prompt: fix_prompt.clone(),
6945                spec,
6946                seat,
6947                cwd: wt.clone(),
6948                timeout,
6949                allow_write: true,
6950                sessions,
6951                artifacts: artifacts.clone(),
6952                stem: "fix-recovery".to_owned(),
6953                handover: None,
6954            })
6955            .await;
6956        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6957            rec.repair = Repair::WaitingForRestore { before: head };
6958            self.state.recovery = Some(rec);
6959            return self.stop_for_resource();
6960        }
6961        let failed = match &out {
6962            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6963            AgentOutcome::Quota(o) => {
6964                self.state.quota.push(QuotaLoss::from_report(
6965                    seat.key.clone(),
6966                    "fix",
6967                    o.quota.as_ref(),
6968                ));
6969                Some("rate limited (quota); the fixer could not run".to_owned())
6970            }
6971            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6972            AgentOutcome::Failed(e) => Some(e.clone()),
6973        };
6974        self.state.seats.insert(seat.key.clone(), seat);
6975        let coordination = self.coordinate(&wt).await?;
6976        let failed = match failed {
6977            // The agent's partial edits are not kept.
6978            Some(why) => {
6979                if let Err(e) = git::restore_to(&wt, &head).await {
6980                    return self
6981                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6982                        .await;
6983                }
6984                Some(why)
6985            }
6986            None => {
6987                if let Ok(r) = git::rescue_commit(
6988                    &wt,
6989                    "magi: recovery repair of the failed verification (uncommitted work)",
6990                )
6991                .await
6992                {
6993                    self.state.note_withheld("fix", &r.withheld);
6994                }
6995                None
6996            }
6997        };
6998        let after = git::rev_parse(&wt, "HEAD").await?;
6999        drop(coordination);
7000        self.state.event(
7001            "recovery",
7002            match &failed {
7003                Some(why) => format!("repair: nothing kept ({why})"),
7004                None if after != head => {
7005                    format!("repair: committed {}", short(&after))
7006                }
7007                None => "repair: the fixer made no commit".to_owned(),
7008            },
7009        );
7010        rec.repair = Repair::Done {
7011            before: head.clone(),
7012            committed: after != head,
7013            after,
7014            failed,
7015        };
7016        self.state.recovery = Some(rec);
7017        self.state.save()?;
7018        Ok(())
7019    }
7020
7021    /// A repair whose rollback could not be confirmed: the run is blocked and
7022    /// resumable, and the unconfirmed state is never reviewed.
7023    async fn stop_recovery(
7024        &mut self,
7025        mut rec: crate::recovery::RecoveryRecord,
7026        head: &str,
7027        why: String,
7028    ) -> Result<()> {
7029        // `Started` stays recorded: a resume retries the restore.
7030        rec.repair = crate::recovery::Repair::Started {
7031            before: head.to_owned(),
7032        };
7033        self.state.recovery = Some(rec);
7034        self.state
7035            .event("recovery", format!("repair could not be undone: {why}"));
7036        self.state.status = RunStatus::Blocked;
7037        self.state.save()?;
7038        bail!("recovery repair could not be rolled back: {why}")
7039    }
7040
7041    async fn review_loop(&mut self) -> Result<()> {
7042        if self.hold_base_sync().await? {
7043            return Ok(());
7044        }
7045        loop {
7046            if self
7047                .state
7048                .base_sync
7049                .as_ref()
7050                .is_some_and(|s| s.conflict.is_some())
7051            {
7052                return Ok(());
7053            }
7054            if self.catchup_repairable() {
7055                let record = self.state.reviews.last().expect("repairable round").clone();
7056                let winner = self
7057                    .state
7058                    .winner()
7059                    .cloned()
7060                    .context("catch-up repair has no winner")?;
7061                if !self.fix_review_round(&winner, record).await? {
7062                    return Ok(());
7063                }
7064            }
7065            self.review_loop_pass().await?;
7066            if !self.catchup_repairable() {
7067                return Ok(());
7068            }
7069        }
7070    }
7071
7072    /// Derive an owed catch-up repair from durable round evidence, including
7073    /// reentry after a crash before the fixer was dispatched.
7074    fn catchup_repairable(&self) -> bool {
7075        self.state.reviews.len() < self.state.config.graph.review_rounds
7076            && self.state.reviews.last().is_some_and(|r| {
7077                !r.clean
7078                    && r.blocking == 0
7079                    && r.fix.is_none()
7080                    && r.e2e_status() == E2eStatus::Failed
7081            })
7082    }
7083
7084    async fn review_loop_pass(&mut self) -> Result<()> {
7085        // A base that would not rebase is a person's decision, not a review
7086        // round: nothing here would change the answer, and reviewers and a
7087        // fixer would be spending real budget on a tree that cannot land
7088        // regardless of what they find.
7089        if self
7090            .state
7091            .base_sync
7092            .as_ref()
7093            .is_some_and(|s| s.conflict.is_some())
7094        {
7095            return Ok(());
7096        }
7097        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
7098        // agent files with `magi task add` name the run that paid for it. The
7099        // prompt overlay is cloned alongside it because the waves borrow it
7100        // while `self` is mutably borrowed by the node's own bookkeeping.
7101        let run_id = self.state.id.clone();
7102        let prompts = self.state.config.prompts.clone();
7103        let Some(winner) = self.state.winner().cloned() else {
7104            return Ok(());
7105        };
7106        let granted_review = self
7107            .state
7108            .repair_grants
7109            .last()
7110            .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7111            .cloned();
7112        let fresh_review = granted_review
7113            .as_ref()
7114            .is_some_and(|g| self.state.reviews.len() == g.review_start);
7115        let max_rounds = granted_review
7116            .as_ref()
7117            .map_or(self.state.config.graph.review_rounds, |g| {
7118                g.review_start.saturating_add(1)
7119            });
7120        // A clean round, an exhausted round budget, or a stalled tree (see
7121        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
7122        // they are recorded — recomputed here, not read off `status`, so a
7123        // reentry into a run that already stopped restates the identical
7124        // verdict instead of silently handing back whatever an earlier node
7125        // in this same walk clobbered `status` to (a solo-candidate
7126        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
7127        // below runs an empty range once the budget is spent, and would
7128        // otherwise fall through without touching `status` at all.
7129        if let Some(status) =
7130            review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7131        {
7132            // A reentry after a crash between the last round's save and
7133            // `stop_reviewing` reaches the hand-off here, not there.
7134            if status == RunStatus::Gating {
7135                self.record_contested_handoff();
7136            }
7137            self.state.status = status;
7138            self.state.save()?;
7139            return Ok(());
7140        }
7141        self.state.status = RunStatus::Reviewing;
7142        // A last recorded round whose own verification never resolved
7143        // (`ResourceBlocked` — the shared build cache, not the patch) is
7144        // never a concluded round, whatever the round budget says: starting
7145        // a fresh round on top of it would spend a whole new reviewer wave
7146        // re-reading an unchanged patch instead of just retrying the one
7147        // check that actually needs it, and once the budget is spent the
7148        // loop below has nothing left to do at all (its range is empty).
7149        // Retry that check directly instead, exactly the same retry
7150        // `stop_reviewing` already does for its own catch-up case.
7151        if !fresh_review
7152            && self.state.reviews.last().is_some_and(|r| {
7153                r.e2e_status() == E2eStatus::ResourceBlocked
7154                    || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7155            })
7156        {
7157            let shell = self.state.config.shell();
7158            return self
7159                .stop_unresolved_review(
7160                    "the last round's own verification never resolved",
7161                    &shell,
7162                    &winner.worktree,
7163                )
7164                .await;
7165        }
7166
7167        if granted_review.is_none() {
7168            self.recovery_repair(&winner).await?;
7169        }
7170        if self.resource_halt()? {
7171            return Ok(());
7172        }
7173
7174        let repo = self.state.repo.clone();
7175        let root = self.state.worktree_root();
7176        let language = self.state.config.graph.language.clone();
7177        let sessions = self.state.config.graph.sessions;
7178        let artifacts = self.merge_repair_artifacts();
7179        let base = self.landing_base();
7180        let base_short = short(&base);
7181        let reviewers = self.roles.reviewers.clone();
7182        let shell = self.state.config.shell();
7183
7184        for round in (self.state.reviews.len() + 1)..=max_rounds {
7185            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7186            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7187            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7188            // The prior round's own record, already persisted — never a
7189            // hand-carried variable of just its failing output: that is
7190            // exactly what let a round's e2e result drift out of sync with
7191            // which commit it was actually about (see `SCHEMA`'s doc for
7192            // schema 8). Judged against `head`, the commit reviewers are
7193            // about to look at now, so the summary always reads as "an
7194            // earlier head" here — this round's own patch has not been
7195            // checked yet.
7196            let prev_verification = self
7197                .state
7198                .reviews
7199                .last()
7200                .and_then(|r| r.verification_summary(&head));
7201            // Only the first round of a run that took a branch over: later
7202            // rounds have this run's own history.
7203            let prior_section = match (&self.state.recovery, round) {
7204                (Some(rec), 1) if self.state.reviews.is_empty() => {
7205                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7206                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7207                            Some(crate::recovery::head_of(
7208                                &git::diff(&winner.worktree, reviewed, "HEAD")
7209                                    .await
7210                                    .unwrap_or_default(),
7211                                crate::recovery::SINCE_DIFF_MAX,
7212                            ))
7213                        }
7214                        _ => None,
7215                    };
7216                    Some(prompt::recovery_review(rec, since.as_deref()))
7217                }
7218                _ => None,
7219            };
7220
7221            // Each reviewer gets its own detached checkout of exactly this
7222            // commit: nobody can perturb the winner's tree, and the fixer can
7223            // keep working without racing a reviewer.
7224            let mut jobs = Vec::new();
7225            let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7226                self.coordinate(&repo).await?
7227            } else {
7228                None
7229            };
7230            for (r, spec) in reviewers.iter().cloned().enumerate() {
7231                let wt = root.join(format!("review-{}", r + 1));
7232                // `clean -fdx` in a reset wipes setup's products, so every
7233                // round sets the seat up again.
7234                if wt.exists() {
7235                    crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7236                } else {
7237                    let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7238                        None
7239                    } else {
7240                        self.coordinate(&repo).await?
7241                    };
7242                    git::worktree_add_detached(&repo, &wt, &head).await?;
7243                }
7244                let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7245                    None
7246                } else {
7247                    self.coordinate(&repo).await?
7248                };
7249                self.setup_seat_worktree(&repo, &wt, false).await?;
7250                let seat_key = format!("review-{}", r + 1);
7251                // The seat starts the round on whoever answered it last, not
7252                // on the agent the spec names, so a failure is not re-paid.
7253                let spec = pick_start_spec(
7254                    &self.roles.reviewer_roster,
7255                    spec,
7256                    self.state.seat_history.get(&seat_key),
7257                );
7258                let seat = self.seat(&seat_key, &spec.id);
7259                jobs.push(SeatJob {
7260                    prompt: prompt::review(&prompt::ReviewCtx {
7261                        instruction: &self.state.instruction,
7262                        branch: &winner.branch,
7263                        base_short: &base_short,
7264                        stat: &stat,
7265                        patch: &patch,
7266                        verification: prev_verification.as_ref(),
7267                        prior: prior_section.as_deref(),
7268                        reviewers: reviewers.len(),
7269                        round,
7270                        rounds: max_rounds,
7271                        // A review-only run has no rankings, so nothing
7272                        // competed for this patch and the reviewer is told so.
7273                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7274                        lens: Lens::for_seat(r),
7275                        language: &language,
7276                        github_language: &self.state.config.graph.github_language,
7277                    }),
7278                    spec,
7279                    seat,
7280                    cwd: wt,
7281                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7282                    allow_write: false,
7283                    sessions,
7284                    artifacts: artifacts.clone(),
7285                    stem: format!("review-{round}-{}", r + 1),
7286                    handover: None,
7287                });
7288            }
7289
7290            drop(coordination);
7291            self.state.event(
7292                "review",
7293                format!(
7294                    "round {round}: {} reviewers on {}",
7295                    jobs.len(),
7296                    short(&head)
7297                ),
7298            );
7299            let mut quota_losses = Vec::new();
7300            let review_retries = self.state.config.graph.retries;
7301            let review_cache = self.state.config.cache_dir();
7302            let ctx = WaveCtx {
7303                carry_seats: true,
7304                pauses: Some((&self.pause, &self.interrupt)),
7305                run: &run_id,
7306                node: "review",
7307                prompts: &prompts,
7308                cache: review_cache.as_deref(),
7309                round: Some(round),
7310            };
7311            let results = ask_json_wave::<Review>(
7312                jobs,
7313                Arc::clone(&self.sem),
7314                review_retries,
7315                &self.roles.reviewer_roster,
7316                &ctx,
7317                &mut quota_losses,
7318                &mut self.state,
7319                &|_: &Review| Ok(()),
7320            )
7321            .await;
7322            if self.state.resource_block.is_some() {
7323                return self.stop_for_resource();
7324            }
7325            // Counted before the move below: how many of *this* round's
7326            // reviewer seats were lost to their own rate limit, as opposed to
7327            // a crash, a timeout, or unparsable output — see `round_is_clean`.
7328            let round_quota_missing = quota_losses.len();
7329            self.state.quota.extend(quota_losses);
7330
7331            let mut records = Vec::new();
7332            let mut all_findings = Vec::new();
7333            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7334                let agent_id = seat.agent.clone();
7335                self.state.seats.insert(seat.key.clone(), seat);
7336                let mut record = ReviewRecord {
7337                    reviewer: r + 1,
7338                    agent: agent_id,
7339                    summary: String::new(),
7340                    findings: Vec::new(),
7341                    vote: None,
7342                    failed: None,
7343                    duration_ms: 0,
7344                    // Set for both outcomes: `failed: Some(_)` with
7345                    // `attempts > 0` is a seat every retry still lost, not a
7346                    // recovered one — only `failed: None` with `attempts > 0`
7347                    // reads as "answered after a nudge" (see this field's own
7348                    // doc).
7349                    attempts,
7350                };
7351                match res {
7352                    Ok((review, out)) => {
7353                        // Sanitized here, at the point every other piece of
7354                        // agent prose in this file is (candidate summaries,
7355                        // deliberation turns, vote reasons): a reviewer's own
7356                        // words are the one thing about it that could name
7357                        // it, and reconsideration below broadcasts this same
7358                        // summary and these same findings to every other
7359                        // seat on the panel.
7360                        record.summary =
7361                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
7362                        record.vote = Some(review.vote);
7363                        record.duration_ms = out.duration_ms;
7364                        for (n, mut f) in review.findings.into_iter().enumerate() {
7365                            // ids are magi's, never the agent's: the fixer's
7366                            // adoption report is keyed by them.
7367                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7368                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7369                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7370                            // `file` is agent-supplied prose too, never
7371                            // checked against the real tree — the same
7372                            // exposure `title`/`detail` above have, just in
7373                            // a field easy to forget because it looks like a
7374                            // path rather than free text.
7375                            f.file = f
7376                                .file
7377                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7378                            all_findings.push(f.clone());
7379                            record.findings.push(f);
7380                        }
7381                        self.state.event(
7382                            "review",
7383                            format!(
7384                                "round {round}: reviewer {} voted {} with {} finding(s)",
7385                                r + 1,
7386                                review.vote.label(),
7387                                record.findings.len()
7388                            ),
7389                        );
7390                    }
7391                    Err(e) => {
7392                        record.failed = Some(e.to_string());
7393                        self.state.event(
7394                            "review",
7395                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7396                        );
7397                    }
7398                }
7399                records.push(record);
7400            }
7401
7402            // Tally the round's votes and, if they split, spend the one
7403            // round of reconsideration the split -> deliberate -> revote
7404            // shape `judge`/`vote` use for the panel, sized down to what a
7405            // read-only review round can afford: one round, and a revote
7406            // rather than an argument, because the panel already wrote its
7407            // reasoning down as findings the first time around.
7408            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7409            let vote_split =
7410                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7411            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7412            if vote_split {
7413                self.state.event(
7414                    "review",
7415                    format!(
7416                        "round {round}: votes split ({}) — one round of reconsideration",
7417                        initial_votes
7418                            .iter()
7419                            .map(|v| v.label())
7420                            .collect::<Vec<_>>()
7421                            .join(", ")
7422                    ),
7423                );
7424                // Seats read every seat's findings and votes, still numbered
7425                // and never named — the same anonymity `review` itself keeps.
7426                let panel: Vec<ReviewSeatReport<'_>> = records
7427                    .iter()
7428                    .filter_map(|r| {
7429                        r.vote.map(|vote| ReviewSeatReport {
7430                            reviewer: r.reviewer,
7431                            vote,
7432                            summary: &r.summary,
7433                            findings: &r.findings,
7434                        })
7435                    })
7436                    .collect();
7437
7438                let mut jobs = Vec::new();
7439                let mut seats_at = Vec::new();
7440                for (r, spec) in reviewers.iter().cloned().enumerate() {
7441                    // A seat with no initial vote has nothing to reconsider
7442                    // from and stays absent, the same as it stayed absent
7443                    // from `panel` above.
7444                    if records[r].vote.is_none() {
7445                        continue;
7446                    }
7447                    let wt = root.join(format!("review-{}", r + 1));
7448                    let seat_key = format!("review-{}", r + 1);
7449                    let spec = self.occupant(&seat_key, spec);
7450                    let seat = self.seat(&seat_key, &spec.id);
7451                    // A seat with no live session has already forgotten the
7452                    // initial review's prompt — restate the patch it is
7453                    // voting on, the same as `deliberate`/`vote` do for a
7454                    // judge in the same position.
7455                    // The panel already carries this seat's own review and
7456                    // vote, so restating the patch makes the prompt whole for
7457                    // a seat handed to another agent.
7458                    let build = |with_patch: bool| {
7459                        prompt::review_reconsider(&ReviewReconsiderCtx {
7460                            instruction: &self.state.instruction,
7461                            reviewer: r + 1,
7462                            lens: Lens::for_seat(r),
7463                            panel: &panel,
7464                            patch: with_patch.then_some(ReviewPatch {
7465                                branch: &winner.branch,
7466                                base_short: &base_short,
7467                                stat: &stat,
7468                                patch: &patch,
7469                            }),
7470                            round,
7471                            rounds: max_rounds,
7472                            language: &language,
7473                        })
7474                    };
7475                    let full = build(true);
7476                    let prompt = if has_context(&spec, &seat, sessions) {
7477                        build(false)
7478                    } else {
7479                        full.clone()
7480                    };
7481                    jobs.push(SeatJob {
7482                        prompt,
7483                        spec,
7484                        seat,
7485                        cwd: wt,
7486                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7487                        allow_write: false,
7488                        sessions,
7489                        artifacts: artifacts.clone(),
7490                        stem: format!("review-{round}-reconsider-{}", r + 1),
7491                        handover: Some(full),
7492                    });
7493                    seats_at.push(r);
7494                }
7495
7496                let mut recon_quota_losses = Vec::new();
7497                let recon_cache = self.state.config.cache_dir();
7498                let recon_ctx = WaveCtx {
7499                    carry_seats: true,
7500                    pauses: Some((&self.pause, &self.interrupt)),
7501                    run: &run_id,
7502                    node: "review",
7503                    prompts: &prompts,
7504                    cache: recon_cache.as_deref(),
7505                    round: Some(round),
7506                };
7507                let recon_results = ask_json_wave::<ReviewRevote>(
7508                    jobs,
7509                    Arc::clone(&self.sem),
7510                    review_retries,
7511                    &self.roles.reviewer_roster,
7512                    &recon_ctx,
7513                    &mut recon_quota_losses,
7514                    &mut self.state,
7515                    &|_: &ReviewRevote| Ok(()),
7516                )
7517                .await;
7518                if self.state.resource_block.is_some() {
7519                    return self.stop_for_resource();
7520                }
7521                self.state.quota.extend(recon_quota_losses);
7522
7523                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7524                    let agent_id = seat.agent.clone();
7525                    self.state.seats.insert(seat.key.clone(), seat);
7526                    let mut rec = ReviewRevoteRecord {
7527                        reviewer: r + 1,
7528                        agent: agent_id,
7529                        vote: None,
7530                        reason: String::new(),
7531                        failed: None,
7532                    };
7533                    match res {
7534                        Ok((rv, _)) => {
7535                            rec.vote = Some(rv.vote);
7536                            rec.reason =
7537                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7538                            self.state.event(
7539                                "review",
7540                                format!(
7541                                    "round {round}: reviewer {} revoted {}",
7542                                    r + 1,
7543                                    rv.vote.label()
7544                                ),
7545                            );
7546                        }
7547                        Err(e) => {
7548                            rec.failed = Some(e.to_string());
7549                            self.state.event(
7550                                "review",
7551                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7552                            );
7553                        }
7554                    }
7555                    reconsideration.push(rec);
7556                }
7557            } else if initial_votes.len() > 1 {
7558                self.state.event(
7559                    "review",
7560                    format!(
7561                        "round {round}: votes agreed ({}) — no reconsideration",
7562                        initial_votes[0].label()
7563                    ),
7564                );
7565            }
7566
7567            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7568            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7569            // A round that already has a blocking finding and a round left to
7570            // try is going back to the fixer no matter what `verify.e2e`
7571            // says, so running it first only spends the loop's slowest step
7572            // (minutes, for a Rust repo's full test suite) on a head about
7573            // to be rewritten. Deferred, never skipped: `verify.e2e` still
7574            // runs once a round has no blocking findings left (see
7575            // `round_is_clean`, which a deferred — empty — `e2e` can never
7576            // satisfy since `blocking` is nonzero whenever this branch is
7577            // taken), and `stop_reviewing` forces a real run before it will
7578            // ever read a deferred round as green.
7579            let defer_e2e =
7580                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7581            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7582                let reason =
7583                    format!("{blocking} blocking finding(s) already required a fix this round");
7584                self.state.event(
7585                    "verify",
7586                    format!(
7587                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7588                         {}); it will run once a round has none left",
7589                        short(&head)
7590                    ),
7591                );
7592                (Vec::new(), false, true, Some(reason))
7593            } else {
7594                let e2e_commands = self.state.config.verify.e2e.clone();
7595                let cache_dir = self.state.config.cache_dir();
7596                let context = format!("round {round}");
7597                let wait = self.repository_wait();
7598                let (e2e, verify_retried) = with_cache_lease(
7599                    &mut self.state,
7600                    cache_dir.as_deref(),
7601                    "e2e",
7602                    "e2e",
7603                    &winner.worktree,
7604                    &head,
7605                    verify_timeout,
7606                    &context,
7607                    &wait,
7608                    |state, budget| {
7609                        let shell = shell.clone();
7610                        let e2e_commands = e2e_commands.clone();
7611                        let worktree = winner.worktree.clone();
7612                        let context = context.clone();
7613                        async move {
7614                            run_e2e_with_retry(
7615                                state,
7616                                &shell,
7617                                &e2e_commands,
7618                                &worktree,
7619                                budget,
7620                                &context,
7621                            )
7622                            .await
7623                        }
7624                    },
7625                )
7626                .await?;
7627                (e2e, verify_retried, false, None)
7628            };
7629
7630            let expected = records.len();
7631            let answered = records.iter().filter(|r| r.failed.is_none()).count();
7632            let incomplete = answered < expected;
7633            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7634            let policy = self.state.config.graph.incomplete_review;
7635            let clean = round_is_clean(
7636                blocking,
7637                e2e_ok,
7638                answered,
7639                expected,
7640                round_quota_missing,
7641                policy,
7642            );
7643
7644            let mut round_record = ReviewRound {
7645                round,
7646                head: head.clone(),
7647                verified_head: None,
7648                verified_at: None,
7649                reviews: records,
7650                e2e,
7651                verify_retried,
7652                e2e_deferred,
7653                e2e_defer_reason,
7654                fix: None,
7655                blocking,
7656                answered,
7657                expected,
7658                clean,
7659                progressed: false,
7660                vote_split,
7661                reconsideration,
7662                verdict: None,
7663            };
7664            // The final vote per seat is its revote where reconsideration
7665            // ran and answered, its initial vote otherwise — the same
7666            // fallback `tally` uses for a judge whose private vote failed.
7667            round_record.verdict = ReviewVote::worst(
7668                round_record
7669                    .final_votes()
7670                    .into_iter()
7671                    .map(|(_, _, vote)| vote),
7672            );
7673            // Which commit and when magi actually attempted to check —
7674            // known the moment a command was dispatched against `head`,
7675            // whether or not it finished: a resource-blocked attempt still
7676            // targeted a specific commit at a specific time, and leaving
7677            // that unrecorded is exactly what made `verification_summary`
7678            // report a fresh attempt as "commit unknown ... recorded before
7679            // this was tracked", indistinguishable from a genuinely old,
7680            // untracked record. Only a deferred or unconfigured round never
7681            // ran at all and has nothing to record — see
7682            // `ReviewRound::verified_head`'s own doc.
7683            if !matches!(
7684                round_record.e2e_status(),
7685                E2eStatus::Deferred | E2eStatus::NotConfigured
7686            ) {
7687                round_record.verified_head = Some(head.clone());
7688                round_record.verified_at = Some(Timestamp::now());
7689            }
7690
7691            if incomplete {
7692                let missing: Vec<String> = round_record
7693                    .reviews
7694                    .iter()
7695                    .filter(|r| r.failed.is_some())
7696                    .map(|r| format!("review-{}", r.reviewer))
7697                    .collect();
7698                self.state.event(
7699                    "review",
7700                    format!(
7701                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7702                        missing.join(", ")
7703                    ),
7704                );
7705            }
7706
7707            if clean {
7708                self.state.event(
7709                    "review",
7710                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7711                        format!(
7712                            "round {round}: clean (warn policy, incomplete panel) — no \
7713                             blocking findings from the seats that answered, verification green"
7714                        )
7715                    } else if incomplete {
7716                        format!(
7717                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7718                             quorum) — no blocking findings from the seats that answered, \
7719                             verification green",
7720                            expected - answered
7721                        )
7722                    } else {
7723                        format!("round {round}: clean — no blocking findings, verification green")
7724                    },
7725                );
7726                self.state.reviews.push(round_record);
7727                self.state.status = RunStatus::Gating;
7728                self.state.save()?;
7729                return Ok(());
7730            }
7731
7732            // Nothing was raised and verification passed, but not every seat
7733            // answered and `round_is_clean` still refused to call it clean —
7734            // either a seat is missing for a reason other than its own quota
7735            // (a crash, a timeout, unparsable output — worth another try), or
7736            // every seat that could have answered lost its quota and nobody
7737            // is left to decide on: re-review rather than send the fixer
7738            // after a round with nothing to fix.
7739            if incomplete && blocking == 0 && e2e_ok {
7740                self.state.reviews.push(round_record);
7741                self.state.save()?;
7742                if round == max_rounds {
7743                    self.state.status = RunStatus::Blocked;
7744                    self.state.event(
7745                        "review",
7746                        format!(
7747                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7748                             refusing to call it clean",
7749                            expected - answered
7750                        ),
7751                    );
7752                    return Ok(());
7753                }
7754                continue;
7755            }
7756
7757            // Nothing for the fixer to act on (`blocking == 0`) and the only
7758            // reason this round is not clean is that magi itself never got
7759            // a command to run — the shared build cache, not the patch (see
7760            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7761            // the fixer would invite a change to appease contention that has
7762            // nothing to do with the diff, and would leave this attempt
7763            // sitting in the next round's prompt as if it were about an
7764            // earlier, superseded commit rather than what it actually is:
7765            // the same head, still waiting to be checked. Wait for it the
7766            // same way the final round's own contention is already handled,
7767            // whatever round this happens to be.
7768            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7769                self.state.reviews.push(round_record);
7770                return self
7771                    .stop_unresolved_review(
7772                        "the round's own verification could not run",
7773                        &shell,
7774                        &winner.worktree,
7775                    )
7776                    .await;
7777            }
7778
7779            if round == max_rounds {
7780                self.state.reviews.push(round_record);
7781                return self
7782                    .stop_reviewing(
7783                        &format!(
7784                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7785                        ),
7786                        &shell,
7787                        &winner.worktree,
7788                    )
7789                    .await;
7790            }
7791
7792            if !self.fix_review_round(&winner, round_record).await? {
7793                return Ok(());
7794            }
7795        }
7796        Ok(())
7797    }
7798
7799    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7800    /// False means admission or continuation policy stopped the review loop.
7801    async fn fix_review_round(
7802        &mut self,
7803        winner: &Candidate,
7804        mut round_record: ReviewRound,
7805    ) -> Result<bool> {
7806        let round = round_record.round;
7807        let shell = self.state.config.shell();
7808        let step = self.run_fix_round(winner, &mut round_record).await?;
7809        if matches!(step, FixStep::Resource) {
7810            self.stop_for_resource()?;
7811            return Ok(false);
7812        }
7813        if self.state.reviews.len() < round {
7814            self.state.reviews.push(round_record);
7815        } else {
7816            self.state.reviews[round - 1] = round_record;
7817        }
7818        self.state.save()?;
7819
7820        // The fixer's own report never came back this round, even after
7821        // `continue_fix_report`'s own budget was spent on it — not an
7822        // ordinary "no report" (dropped stream, quota, plain failure),
7823        // which already reads that way and is left to the existing round
7824        // budget. Stopping here, rather than opening another round, is
7825        // what keeps a next reviewer/fixer wave from ever being
7826        // dispatched onto `winner.worktree` while whatever the seat's
7827        // last call may still have running there is unaccounted for: no
7828        // process liveness check exists (and none is being added — see
7829        // AGENTS.md/this task's own scope), so the only way to honour
7830        // "nothing starts before a valid report returns" is to not start
7831        // anything further on this worktree from this run at all.
7832        if matches!(step, FixStep::Recorded { report_lost: true }) {
7833            self.stop_reviewing(
7834                "the fixer's adoption report never came back, even after resuming its \
7835                         own seat; refusing to start another round against the same worktree \
7836                         while that is unresolved",
7837                &shell,
7838                &winner.worktree,
7839            )
7840            .await?;
7841            return Ok(false);
7842        }
7843
7844        let streak = self
7845            .state
7846            .reviews
7847            .iter()
7848            .rev()
7849            .take_while(|r| !r.progressed)
7850            .count();
7851        if streak >= STAGNANT_LIMIT {
7852            self.stop_reviewing(
7853                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7854                &shell,
7855                &winner.worktree,
7856            )
7857            .await?;
7858            return Ok(false);
7859        }
7860        Ok(true)
7861    }
7862
7863    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7864    async fn run_fix_round(
7865        &mut self,
7866        winner: &Candidate,
7867        round_record: &mut ReviewRound,
7868    ) -> Result<FixStep> {
7869        let round = round_record.round;
7870        let max_rounds = self.state.config.graph.review_rounds;
7871        let run_id = self.state.id.clone();
7872        let prompts = self.state.config.prompts.clone();
7873        let language = self.state.config.graph.language.clone();
7874        let sessions = self.state.config.graph.sessions;
7875        let artifacts = self.merge_repair_artifacts();
7876        let base = self.landing_base();
7877        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7878        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7879        let this_round_verification = round_record.verification_summary(&head);
7880        let all_findings: Vec<_> = round_record
7881            .reviews
7882            .iter()
7883            .flat_map(|r| r.findings.iter().cloned())
7884            .collect();
7885        // Fix. The winner's own implementer seat continues its conversation:
7886        // the competition is over, so context is pure benefit now.
7887        let blocking_findings: Vec<_> = all_findings
7888            .iter()
7889            .filter(|f| f.severity.blocks())
7890            .cloned()
7891            .collect();
7892        let fix_prompt = prompt::fix_in(
7893            &self.state.instruction,
7894            &blocking_findings,
7895            this_round_verification.as_ref(),
7896            round,
7897            max_rounds,
7898            &language,
7899            &self.state.config.graph.github_language,
7900        );
7901        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7902        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7903        let (job, seat, out) = self
7904            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7905                prompt: fix_prompt.clone(),
7906                spec,
7907                seat,
7908                cwd: winner.worktree.clone(),
7909                timeout,
7910                allow_write: true,
7911                sessions,
7912                artifacts: artifacts.clone(),
7913                stem: format!("fix-{round}"),
7914                handover: None,
7915            })
7916            .await;
7917        // Not asked: the round is not recorded (so a resume repeats it),
7918        // no fixer was handed over to, and the seat is left as it was.
7919        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7920            return Ok(FixStep::Resource);
7921        }
7922        let agent_id = seat.agent.clone();
7923
7924        let mut fix = FixRecord {
7925            agent: agent_id,
7926            addressed: Vec::new(),
7927            rejected: Vec::new(),
7928            notes: String::new(),
7929            committed: false,
7930            failed: None,
7931            duration_ms: 0,
7932            continuation: None,
7933        };
7934        let mut continuation = ContinuationRecord::not_needed();
7935        let mut final_seat = seat.clone();
7936        match out {
7937            AgentOutcome::Ok(o) => {
7938                fix.duration_ms = o.duration_ms;
7939                let parsed = verdict::extract_json::<FixReport>(&o.text);
7940                // A parsed report standing next to a command this same
7941                // reply's own CLI never confirmed the exit status of is
7942                // not a resolved answer — the identical `CommandEvidence`
7943                // `state.jobs` renders, read here instead of only on
7944                // display, per the completion judgment and the shown
7945                // record needing to agree.
7946                let incomplete_reason = match &parsed {
7947                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7948                        "the reply parsed, but it reported a command whose own CLI never \
7949                             confirmed an exit status"
7950                            .to_owned(),
7951                    ),
7952                    Ok(_) => None,
7953                    Err(e) => Some(e.to_string()),
7954                };
7955                match incomplete_reason {
7956                    None => {
7957                        let report = parsed.expect("checked Ok above");
7958                        fix.addressed = report.addressed;
7959                        fix.rejected = report.rejected;
7960                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7961                    }
7962                    Some(reason) => {
7963                        let (resumed_seat, resolved, failure, cont) = self
7964                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7965                            .await;
7966                        fix.duration_ms += cont.cumulative_wait_ms;
7967                        continuation = cont;
7968                        final_seat = resumed_seat;
7969                        match resolved {
7970                            Some(report) => {
7971                                fix.addressed = report.addressed;
7972                                fix.rejected = report.rejected;
7973                                fix.notes =
7974                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
7975                            }
7976                            None => fix.failed = failure,
7977                        }
7978                    }
7979                }
7980            }
7981            // The CLI's raw error JSON is not a fix report to parse.
7982            AgentOutcome::Dropped(o) => {
7983                fix.duration_ms = o.duration_ms;
7984                let why = o
7985                    .dropped
7986                    .as_ref()
7987                    .map(|d| d.why.as_str())
7988                    .unwrap_or("the CLI ended the stream without delivering its answer");
7989                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7990            }
7991            AgentOutcome::Quota(o) => {
7992                self.state.quota.push(QuotaLoss::from_report(
7993                    final_seat.key.clone(),
7994                    "fix",
7995                    o.quota.as_ref(),
7996                ));
7997                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7998            }
7999            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8000        }
8001        if self.state.resource_block.is_some() {
8002            self.state.seats.insert(final_seat.key.clone(), final_seat);
8003            return Ok(FixStep::Resource);
8004        }
8005        fix.continuation = Some(continuation);
8006        self.state.seats.insert(final_seat.key.clone(), final_seat);
8007        let coordination = self.coordinate(&winner.worktree).await?;
8008        if let Ok(r) = git::rescue_commit(
8009            &winner.worktree,
8010            &format!("magi: review round {round} fixes (uncommitted work)"),
8011        )
8012        .await
8013        {
8014            self.state.note_withheld("fix", &r.withheld);
8015        }
8016        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8017        fix.committed = after != before;
8018        // Judged by what `git` says moved against base, never by the
8019        // fixer's own `addressed`/`rejected` count — see
8020        // `ReviewRound::progressed`. Propagated with `?`, the same as the
8021        // `patch` snapshot above: swallowing this error would default
8022        // `diff_after` to empty, which almost always differs from a
8023        // non-empty `patch` and reads as "progressed" — exactly backwards
8024        // for a `git` failure the stagnation check cannot see through.
8025        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8026        drop(coordination);
8027        let progressed = diff_after != patch;
8028        let commit_note = if fix.committed {
8029            "committed"
8030        } else {
8031            "NO new commit"
8032        };
8033        let tree_note = if progressed {
8034            "changed vs base"
8035        } else {
8036            "unchanged vs base"
8037        };
8038        self.state.event(
8039            "fix",
8040            match &fix.failed {
8041                // Distinct on purpose from "0 addressed, 0 rejected": the
8042                // fixer's own diff still landed (blocking counts do keep
8043                // falling round over round), only its adoption report did
8044                // not come back, so this must never read like every
8045                // finding was reviewed and declined.
8046                Some(reason) => {
8047                    format!(
8048                        "round {round}: fixer's adoption report was lost ({reason}); \
8049                             {commit_note}, tree {tree_note}"
8050                    )
8051                }
8052                None => format!(
8053                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8054                         {tree_note}{}",
8055                    fix.addressed.len(),
8056                    fix.rejected.len(),
8057                    if continuation.outcome == ContinuationOutcome::Resumed {
8058                        format!(
8059                            " (adoption report recovered after {} continuation(s))",
8060                            continuation.attempts
8061                        )
8062                    } else {
8063                        String::new()
8064                    },
8065                ),
8066            },
8067        );
8068        round_record.fix = Some(fix);
8069        round_record.progressed = progressed;
8070        Ok(FixStep::Recorded {
8071            report_lost: matches!(
8072                continuation.outcome,
8073                ContinuationOutcome::Exhausted
8074                    | ContinuationOutcome::QuotaLost
8075                    | ContinuationOutcome::NoSession
8076            ),
8077        })
8078    }
8079
8080    /// Decide, from the last recorded round's own verification, whether
8081    /// stopping the review loop is a hand-off or a genuine block.
8082    ///
8083    /// Called once the loop has given up trying — the round budget is spent,
8084    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
8085    /// findings still open, never while a round is still clean or the
8086    /// incomplete-panel case handled inline above. Gate and e2e are facts
8087    /// about the tree; a lingering review finding is an opinion, and this
8088    /// workload's own `magi stats` puts reviewer precision low enough
8089    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
8090    /// must not by itself stand between a green, verified change and the
8091    /// human who decides what to do with it. A red e2e is not an opinion, so
8092    /// that case still blocks, with the failing command and a tail of its
8093    /// output recorded here rather than left in `run.json` for someone to go
8094    /// find.
8095    ///
8096    /// A round that deferred its own e2e (see [`Config::graph`]'s
8097    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
8098    /// only because nothing ran, and treating an empty list as a passing one
8099    /// here is exactly the "deferred painted green" bug this function exists
8100    /// to not have. When the last round's own verification never resolved —
8101    /// deferred on purpose, or a real attempt the shared build cache blocked
8102    /// — this makes (or retries) the real run, on the actual worktree this
8103    /// loop is about to stop touching, before deciding anything. A
8104    /// resource-blocked attempt is likewise never read as either green or
8105    /// red: it is evidence about the machine, not the patch (see
8106    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
8107    /// blocked cache leaves this call without deciding rather than guessing
8108    /// — the caller retries on a later reentry.
8109    /// Record, once, that the review loop handed off over a blocking finding
8110    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
8111    /// `land` asks the owner even with `land_approval` off. Called from every
8112    /// path that concludes `Gating`; a reentry keeps the first record.
8113    fn record_contested_handoff(&mut self) {
8114        if self.state.contested_handoff.is_some() {
8115            return;
8116        }
8117        let Some(contested) = self
8118            .state
8119            .reviews
8120            .last()
8121            .and_then(ReviewRound::contested_handoff)
8122        else {
8123            return;
8124        };
8125        self.state.event(
8126            "review",
8127            format!(
8128                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8129                 wait for the owner's approval",
8130                contested.findings.len(),
8131                contested.rejecters.len()
8132            ),
8133        );
8134        self.state.contested_handoff = Some(contested);
8135    }
8136
8137    async fn stop_unresolved_review(
8138        &mut self,
8139        why: &str,
8140        shell: &[String],
8141        worktree: &Path,
8142    ) -> Result<()> {
8143        self.stop_reviewing_inner(why, shell, worktree, true).await
8144    }
8145
8146    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8147        self.stop_reviewing_inner(why, shell, worktree, false).await
8148    }
8149
8150    async fn stop_reviewing_inner(
8151        &mut self,
8152        why: &str,
8153        shell: &[String],
8154        worktree: &Path,
8155        repairable: bool,
8156    ) -> Result<()> {
8157        let round_idx = self.state.reviews.len() - 1;
8158        // A deferred round and a resource-blocked one are the same shape
8159        // here: neither has a real result yet, and both get one more
8160        // attempt. Read off `e2e_status` — the single source for this —
8161        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
8162        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
8163        // still retries instead of being read as a settled result the
8164        // instant it stops being empty.
8165        let needs_catchup_run = matches!(
8166            self.state.reviews[round_idx].e2e_status(),
8167            E2eStatus::Deferred | E2eStatus::ResourceBlocked
8168        );
8169        if needs_catchup_run {
8170            let round = self.state.reviews[round_idx].round;
8171            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8172            let commands = self.state.config.verify.e2e.clone();
8173            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8174            let cache_dir = self.state.config.cache_dir();
8175            let context = format!(
8176                "round {round}: verification unresolved, catching up before the final decision"
8177            );
8178            let wait = self.repository_wait();
8179            let (outcomes, verify_retried) = with_cache_lease(
8180                &mut self.state,
8181                cache_dir.as_deref(),
8182                "e2e",
8183                "e2e",
8184                worktree,
8185                &attempted_head,
8186                timeout,
8187                &context,
8188                &wait,
8189                |state, budget| {
8190                    let shell = shell.to_vec();
8191                    let commands = commands.clone();
8192                    let context = context.clone();
8193                    async move {
8194                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8195                            .await
8196                    }
8197                },
8198            )
8199            .await?;
8200            if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8201                self.state.record_base_progress(&attempted_head);
8202            }
8203            let last = &mut self.state.reviews[round_idx];
8204            last.e2e = outcomes;
8205            last.verify_retried = verify_retried;
8206            // Always the commit and time this attempt actually targeted,
8207            // whether or not it happens to equal the reviewed `head` and
8208            // whether or not a command finished — see
8209            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
8210            // attempt is recorded too, so a later reader sees "attempted
8211            // again at T2" rather than silence.
8212            last.verified_head = Some(attempted_head);
8213            last.verified_at = Some(Timestamp::now());
8214            if verify_inconclusive(&last.e2e) {
8215                // Still not a real result: `e2e_deferred` is left exactly
8216                // as it was, so `needs_catchup_run` above reads
8217                // `ResourceBlocked` (via `e2e_status`, which checks
8218                // `resource_blocked` before `e2e_deferred`) and retries
8219                // again on the next reentry, rather than recording
8220                // contention as a red e2e and blocking the run on it.
8221                self.state.save()?;
8222                return Ok(());
8223            }
8224            last.e2e_deferred = false;
8225            // A complete panel may have been saved as unclean solely because
8226            // verification could not run. Refresh that verdict with the new
8227            // outcomes before a granted repair consumes the round's result.
8228            // Incomplete panels retain their original policy/quota decision.
8229            if last.answered == last.expected {
8230                last.clean = round_is_clean(
8231                    last.blocking,
8232                    last.e2e.iter().all(CommandOutcome::ok),
8233                    last.answered,
8234                    last.expected,
8235                    0,
8236                    self.state.config.graph.incomplete_review,
8237                );
8238            }
8239        }
8240        let last = &self.state.reviews[round_idx];
8241        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8242
8243        match last.e2e_status() {
8244            E2eStatus::Failed => {
8245                let red: Vec<String> = last
8246                    .e2e
8247                    .iter()
8248                    .filter(|o| !o.ok())
8249                    .map(|o| {
8250                        format!(
8251                            "`{}` -> {:?}\n{}",
8252                            o.command,
8253                            o.code,
8254                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8255                        )
8256                    })
8257                    .collect();
8258                let can_repair = last.blocking == 0 && last.fix.is_none();
8259                self.state.reviews[round_idx].clean = false;
8260                self.state
8261                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8262                self.state.status = RunStatus::Blocked;
8263                if repairable
8264                    && can_repair
8265                    && self.state.reviews.len() < self.state.config.graph.review_rounds
8266                {
8267                    self.state.status = RunStatus::Reviewing;
8268                }
8269            }
8270            // `needs_catchup_run` above already retried once this call; if
8271            // it is still blocked, this is magi's own admission it could
8272            // not get a command to run, never a verdict on the patch — the
8273            // run is left exactly where a later reentry can retry again.
8274            E2eStatus::ResourceBlocked => {
8275                self.state.event(
8276                    "review",
8277                    format!(
8278                        "{why}; e2e could not run (resource unavailable); not \
8279                         deciding yet"
8280                    ),
8281                );
8282            }
8283            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8284                self.state.event(
8285                    "review",
8286                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8287                );
8288                self.record_contested_handoff();
8289                self.state.status = RunStatus::Gating;
8290            }
8291        }
8292        self.state.save()?;
8293        Ok(())
8294    }
8295
8296    // ----------------------------------------------------------------- gate
8297
8298    async fn gate(&mut self) -> Result<()> {
8299        if self.hold_base_sync().await? {
8300            return Ok(());
8301        }
8302        // Judged by the review record itself, not by `status`: a solo
8303        // candidate's `judge`/`deliberate` skip rewrites `status` on every
8304        // reentry (see `judge`), and trusting it here is exactly how a run
8305        // that exhausted its review budget got gated and merged a second
8306        // time around. `review_conclusion` recomputes the review loop's own
8307        // verdict from the round records themselves — `Gating` for a clean
8308        // round or a hand-off (see `stop_reviewing`), anything else means the
8309        // loop is still going or genuinely blocked.
8310        // A base the winner could not be replayed onto is a decision, not a
8311        // round: there is no landing tree to gate. Read as its own record for
8312        // the same reason the review verdict is.
8313        if self.state.status == RunStatus::Failed
8314            || self
8315                .state
8316                .base_sync
8317                .as_ref()
8318                .is_some_and(|s| s.conflict.is_some())
8319            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8320                != Some(RunStatus::Gating)
8321        {
8322            return Ok(());
8323        }
8324        let Some(winner) = self.state.winner().cloned() else {
8325            return Ok(());
8326        };
8327        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8328        self.state.reconcile_verification_head(&head);
8329        self.state.save()?;
8330        let wait = self.repository_wait();
8331        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8332            return Ok(());
8333        }
8334        if self.state.gate_ran {
8335            // `review_loop` derives its conclusion from the clean review
8336            // record on every reentry and therefore puts a completed run back
8337            // in `Gating`. A recorded gate is a stronger, terminal fact:
8338            // retain its original command output (or lack of any, for a repo
8339            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
8340            // doc) and restore `Blocked` on a real failure rather than
8341            // pretending the command is still running or running it a second
8342            // time. `gate_ran == false` remains the only shape — unattempted,
8343            // or a resource-blocked retry — that may still need to execute a
8344            // command.
8345            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8346                self.state.status = RunStatus::Blocked;
8347                self.state.save()?;
8348            }
8349            return Ok(());
8350        }
8351        self.state.status = RunStatus::Gating;
8352        let mut outcomes = self.run_gate(&winner).await?;
8353        let mut attempted_head = self.state.gate_head.clone();
8354        loop {
8355            // A resource-blocked outcome means the gate command never actually
8356            // ran - the shared build cache could not be acquired or confirmed
8357            // fresh in time - which is evidence about the machine, not about
8358            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
8359            // Recording it as a red gate would mark a run `Blocked` on nothing
8360            // but contention magi has already logged; leaving `self.state.gate`
8361            // empty and `self.state.gate_ran` false instead keeps the shape
8362            // this function already treats as "still needs to run" (see the
8363            // early-return above), so the next call retries the command
8364            // rather than concluding anything.
8365            if verify_inconclusive(&outcomes) {
8366                self.state.save()?;
8367                return Ok(());
8368            }
8369            if outcomes.iter().all(CommandOutcome::ok) {
8370                break;
8371            }
8372            let fixes_before = self.state.gate_fixes.len();
8373            match self.gate_fix_round(&winner, &outcomes).await? {
8374                GateFix::Retry => {
8375                    outcomes = self.run_gate(&winner).await?;
8376                    attempted_head = self.state.gate_head.clone();
8377                }
8378                GateFix::Stop => {
8379                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8380                    if self.state.gate_fixes.len() > fixes_before
8381                        && attempted_head.as_deref() != Some(current.as_str())
8382                        && verify_current_head(&mut self.state, &winner, &wait).await?
8383                    {
8384                        // A new SHA owes a gate even when a spent fixer changed only
8385                        // commit metadata. This does not grant another repair round.
8386                        outcomes = self.run_gate(&winner).await?;
8387                        attempted_head = self.state.gate_head.clone();
8388                    } else {
8389                        break;
8390                    }
8391                }
8392                GateFix::Defer => {
8393                    self.state.save()?;
8394                    return Ok(());
8395                }
8396            }
8397        }
8398        let passed = outcomes.iter().all(CommandOutcome::ok);
8399        self.state.gate = outcomes;
8400        self.state.gate_ran = true;
8401        self.state.gate_head = attempted_head;
8402        if !self.state.gate.is_empty()
8403            && !verify_inconclusive(&self.state.gate)
8404            && let Some(head) = self.state.gate_head.clone()
8405        {
8406            self.state.record_base_progress(&head);
8407        }
8408        self.state.save()?;
8409        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8410            return Ok(());
8411        }
8412        if !passed {
8413            self.state.status = RunStatus::Blocked;
8414            let spent = self.state.gate_fixes.len();
8415            self.state.event(
8416                "gate",
8417                if spent == 0 {
8418                    "gate failed; not merging".to_owned()
8419                } else {
8420                    format!("gate failed after {spent} gate-fix round(s); not merging")
8421                },
8422            );
8423        }
8424        self.state.save()?;
8425        Ok(())
8426    }
8427
8428    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
8429    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
8430    /// after review is clean and never on a candidate awaiting judging.
8431    ///
8432    /// Never fails the run: a non-zero exit or timeout is a warning and a
8433    /// recorded outcome, and the gate remains the single arbiter. Nothing
8434    /// configured means nothing happens - no event, no commit. `commit_all`
8435    /// commits any leftover change under the neutral identity and returns
8436    /// `false` when the tree is clean, so no empty commit is ever made.
8437    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8438        let commands = state.config.verify.pre_gate.clone();
8439        if commands.is_empty() {
8440            return;
8441        }
8442        let shell = state.config.shell();
8443        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8444        let (outcomes, _) = run_commands(
8445            state,
8446            "pre_gate",
8447            "pre_gate",
8448            0,
8449            &shell,
8450            &commands,
8451            &winner.worktree,
8452            timeout,
8453        )
8454        .await;
8455        for o in &outcomes {
8456            if !o.ok() {
8457                tracing::warn!(
8458                    "pre_gate `{}` failed ({:?}); the gate decides",
8459                    o.command,
8460                    o.code
8461                );
8462            }
8463            state.event(
8464                "pre_gate",
8465                format!(
8466                    "`{}` -> {}",
8467                    o.command,
8468                    if o.ok() {
8469                        "pass".to_owned()
8470                    } else {
8471                        format!(
8472                            "FAIL ({:?})\n{}",
8473                            o.code,
8474                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8475                        )
8476                    }
8477                ),
8478            );
8479        }
8480        state.pre_gate = outcomes;
8481        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8482            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8483                Ok(head) => {
8484                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8485                    state.pre_gate_commit = Some(head);
8486                }
8487                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8488            },
8489            Ok(false) => {}
8490            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8491        }
8492        if let Err(e) = state.save() {
8493            tracing::warn!("could not persist the pre_gate record: {e:#}");
8494        }
8495    }
8496
8497    /// Run `verify.gate` once against the winner's current tree, logging one
8498    /// event per command. Empty when nothing is configured.
8499    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8500        let wait = self.repository_wait();
8501        run_landing_gate(&mut self.state, winner, &wait).await
8502    }
8503
8504    /// One bounded fix round for a failing gate.
8505    ///
8506    /// The fixer is told the failure came from the gate itself, not from a
8507    /// reviewer, and is shown the failed commands, their exit codes and a tail
8508    /// of their output - whatever `[verify].gate` holds, nothing here knows
8509    /// what those commands run. Only a normal non-zero exit that printed
8510    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
8511    /// command or a full disk says nothing about the code, and a fixer sent
8512    /// after it can only appease the machine. The round is judged by what git
8513    /// says moved, never by the fixer's own report, and `verify.e2e` runs
8514    /// again before the gate does, so a fix cannot trade a green gate for a
8515    /// red e2e unnoticed.
8516    async fn gate_fix_round(
8517        &mut self,
8518        winner: &Candidate,
8519        outcomes: &[CommandOutcome],
8520    ) -> Result<GateFix> {
8521        let cap = if self.state.repair_grants.last().is_some_and(|g| {
8522            g.phase == crate::repair_budget::Phase::Fixing
8523                && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8524        }) {
8525            self.state.gate_fixes.len().saturating_add(1)
8526        } else {
8527            self.state.config.graph.gate_fix_rounds
8528        };
8529        let spent = self.state.gate_fixes.len();
8530        if spent >= cap {
8531            if cap > 0 {
8532                self.state.event(
8533                    "gate",
8534                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
8535                );
8536            }
8537            return Ok(GateFix::Stop);
8538        }
8539        if !gate_fixable(outcomes) {
8540            self.state.event(
8541                "gate",
8542                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8543                 command or similar); not spending a fix round on it",
8544            );
8545            return Ok(GateFix::Stop);
8546        }
8547        let min_free = self.state.config.disk.min_free_bytes;
8548        if min_free > 0 {
8549            match crate::disk::free_bytes(&winner.worktree) {
8550                Ok(free) if crate::disk::enough_space(free, min_free) => {}
8551                Ok(free) => {
8552                    self.state.event(
8553                        "gate",
8554                        format!(
8555                            "only {free} bytes free ({min_free} required by `[disk] \
8556                             min_free_bytes`); not spending a fix round on a failure the disk \
8557                             may explain"
8558                        ),
8559                    );
8560                    return Ok(GateFix::Stop);
8561                }
8562                Err(e) => {
8563                    self.state.event(
8564                        "gate",
8565                        format!("free disk space could not be measured ({e:#}); no fix round"),
8566                    );
8567                    return Ok(GateFix::Stop);
8568                }
8569            }
8570        }
8571
8572        let attempt = spent + 1;
8573        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8574        let base = self.landing_base();
8575        let fix_prompt = prompt::gate_fix_in(
8576            &self.state.instruction,
8577            &failed,
8578            attempt,
8579            cap,
8580            &self.state.config.graph.language,
8581            &self.state.config.graph.github_language,
8582        );
8583        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8584        let sessions = self.state.config.graph.sessions;
8585        let artifacts = self.merge_repair_artifacts();
8586        self.state.event(
8587            "gate",
8588            format!("gate failed; gate-fix round {attempt} of {cap}"),
8589        );
8590        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8591        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8592        // Persist before invoking the fixer: it may commit and then be
8593        // interrupted before its result (or the following e2e) is saved.
8594        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8595            head: before.clone(),
8596            passed: false,
8597            outcomes: Vec::new(),
8598        });
8599        self.state.save()?;
8600        let (_, seat, out) = self
8601            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8602                prompt: fix_prompt.clone(),
8603                spec,
8604                seat,
8605                cwd: winner.worktree.clone(),
8606                timeout,
8607                allow_write: true,
8608                sessions,
8609                artifacts: artifacts.clone(),
8610                stem: format!("gate-fix-{attempt}"),
8611                handover: None,
8612            })
8613            .await;
8614        // The fixer was never asked: no round is spent (nothing is recorded)
8615        // and a later reentry asks the same agent again.
8616        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8617            return Ok(GateFix::Defer);
8618        }
8619        let mut record = GateFixRecord {
8620            agent: seat.agent.clone(),
8621            failed,
8622            notes: String::new(),
8623            committed: false,
8624            error: None,
8625        };
8626        match out {
8627            AgentOutcome::Ok(o) => {
8628                // A missing report is not a failed fix: the round is judged
8629                // by the tree below, and the report only carries prose.
8630                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8631                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8632                }
8633            }
8634            AgentOutcome::Dropped(_) => {
8635                record.error = Some("the CLI dropped the stream".to_owned());
8636            }
8637            AgentOutcome::Quota(o) => {
8638                self.state.quota.push(QuotaLoss::from_report(
8639                    seat.key.clone(),
8640                    "gate-fix",
8641                    o.quota.as_ref(),
8642                ));
8643                record.error = Some("rate limited (quota); fixer could not run".to_owned());
8644            }
8645            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8646        }
8647        self.state.seats.insert(seat.key.clone(), seat);
8648        let coordination = self.coordinate(&winner.worktree).await?;
8649        if let Ok(r) = git::rescue_commit(
8650            &winner.worktree,
8651            &format!("magi: gate fix {attempt} (uncommitted work)"),
8652        )
8653        .await
8654        {
8655            self.state.note_withheld("gate-fix", &r.withheld);
8656        }
8657        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8658        record.committed = after != before;
8659        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8660        let note = record.error.clone();
8661        self.state.gate_fixes.push(record);
8662        self.state.save()?;
8663        if !changed {
8664            self.state.event(
8665                "gate",
8666                match note {
8667                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8668                    None => format!("gate-fix round {attempt}: the tree did not change"),
8669                },
8670            );
8671            return Ok(GateFix::Stop);
8672        }
8673        self.state.event(
8674            "gate",
8675            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8676        );
8677
8678        drop(coordination);
8679        let wait = self.repository_wait();
8680        check_gate_repair_e2e(&mut self.state, winner, &wait).await
8681    }
8682
8683    // ---------------------------------------------------------------- merge
8684
8685    /// One read-only rewrite by the summary's author; a text the gate still
8686    /// withholds goes to the owner (see `github_text`) and the fixed fallback
8687    /// applies on their word or on silence. `None` means the run is parked on
8688    /// that question and nothing may be pushed or opened yet.
8689    async fn guarded_pr_message(
8690        &mut self,
8691        winner: &Candidate,
8692        facts: Option<&BranchFacts>,
8693        posting: bool,
8694    ) -> Result<Option<PrMessage>> {
8695        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8696        let head = match final_head.as_deref() {
8697            Some(h) => crate::pr_outcome::FinalHead::At(h),
8698            None => crate::pr_outcome::FinalHead::Unknown,
8699        };
8700        // Only a pull request that is about to be posted pays for a summary.
8701        let summarized = if posting {
8702            Some(self.summarize_change(winner, final_head.as_deref()).await)
8703        } else {
8704            None
8705        };
8706        // A busy shared resource is not a permanent cause: publishing an
8707        // "unavailable" description would stick (a later head move never
8708        // edits the body). Stop before anything is pushed or opened; the
8709        // run is `Blocked` and a resume asks the summary again.
8710        if self.state.resource_block.is_some() {
8711            self.state.event(
8712                "pr-summary",
8713                "a shared resource was busy; nothing was published, resume to summarize again",
8714            );
8715            return Ok(None);
8716        }
8717        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8718            anyhow::bail!(
8719                "the final commit moved during public description generation; retry before publishing"
8720            );
8721        }
8722        let note = match (&summarized, &self.state.change_summary) {
8723            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8724            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8725            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8726            (None, _) => ChangeNote::NotAsked,
8727        };
8728        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8729        if !posting {
8730            let identity = crate::scrub::Identity::current();
8731            return Ok(Some(PrMessage {
8732                title: crate::scrub::scrub(&pr.title, &identity),
8733                body: crate::scrub::scrub(&pr.body, &identity),
8734            }));
8735        }
8736        // This very text was already rejected and asked about: no second
8737        // judge call, rewrite or question, just carry the record on.
8738        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8739        if let Some(known) = self
8740            .state
8741            .github_text
8742            .clone()
8743            .filter(|g| g.fingerprint == raw_fp)
8744        {
8745            return self.settle_withheld(winner, &pr, known).await;
8746        }
8747        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8748        let mut violations =
8749            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8750        let identity = crate::scrub::Identity::current();
8751        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8752            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8753            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8754        {
8755            violations.push(crate::github_text::Violation::SensitiveData);
8756        }
8757        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8758        // only a language violation starts one; sensitive data alone goes
8759        // straight to the owner, with the redacted text as the fallback.
8760        let has_language = violations
8761            .iter()
8762            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8763        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8764            self.state.event(
8765                "github-text",
8766                if has_language {
8767                    format!("description rejected: {violations:?}; requesting one rewrite")
8768                } else {
8769                    format!("description rejected: {violations:?}; asking the owner")
8770                },
8771            );
8772            let mut rewritten = false;
8773            let mut rejected_rewrite = None;
8774            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8775            {
8776                let output = text;
8777                if let Some(title) = summary_title(&output) {
8778                    // Judge the redacted candidate that will actually be posted.
8779                    // Sensitive data alone must not discard a useful rewrite.
8780                    let raw_body = summary_without_title(&output);
8781                    let clean_title = crate::scrub::scrub(&title, &identity);
8782                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8783                    let redacted = clean_title != title || body != raw_body;
8784                    let title = clean_title;
8785                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8786                    if !complete {
8787                        self.state.event(
8788                            "github-text",
8789                            "description rewrite omitted required findings",
8790                        );
8791                    }
8792                    if !body.trim().is_empty() {
8793                        body.push_str("\n\n");
8794                        body.push_str(&public_pr_evidence(&self.state, head));
8795                    }
8796                    let marker = format!("magi:run/{}", self.state.id);
8797                    if !body.trim().is_empty() && !body.contains(&marker) {
8798                        body.push_str(&format!("\n\n{marker}"));
8799                    }
8800                    let rewrite = PrMessage {
8801                        title: title.clone(),
8802                        body: body.clone(),
8803                    };
8804                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8805                    let mut failed = crate::github_text::check_for(
8806                        &self.state.config.graph,
8807                        &title,
8808                        &body,
8809                        redecision,
8810                    );
8811                    if (!crate::github_text::shareable(&title)
8812                        || !crate::github_text::shareable(&body))
8813                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8814                    {
8815                        failed.push(crate::github_text::Violation::SensitiveData);
8816                    }
8817                    if complete
8818                        && !title.trim().is_empty()
8819                        && !body.trim().is_empty()
8820                        && !failed.is_empty()
8821                    {
8822                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8823                            title: title.clone(),
8824                            body: body.clone(),
8825                            failed: failed
8826                                .iter()
8827                                .map(|v| crate::github_text::category(*v).to_owned())
8828                                .collect(),
8829                        });
8830                    }
8831                    if complete
8832                        && !title.trim().is_empty()
8833                        && !raw_body.trim().is_empty()
8834                        && failed.is_empty()
8835                    {
8836                        decision = redecision;
8837                        pr = PrMessage { title, body };
8838                        rewritten = true;
8839                        self.state
8840                            .event("github-text", "description rewrite passed");
8841                        if redacted {
8842                            self.state.event(
8843                                "github-text",
8844                                "sensitive data removed from description rewrite",
8845                            );
8846                        }
8847                    }
8848                }
8849            }
8850            if !rewritten {
8851                self.state.event(
8852                    "github-text",
8853                    "description rewrite unavailable or rejected; asking the owner",
8854                );
8855                if let Some(w) =
8856                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8857                {
8858                    // A different text than the one asked about before: that
8859                    // question no longer describes anything.
8860                    self.retire_github_text_question("the withheld text changed");
8861                    let record = crate::run::GithubTextAsk {
8862                        rewrite: rejected_rewrite,
8863                        fingerprint: raw_fp,
8864                        title: w.title,
8865                        body: w.body,
8866                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8867                        question: None,
8868                        asks: 0,
8869                        resolved: false,
8870                        chosen_title: None,
8871                        choice: None,
8872                        publication: None,
8873                    };
8874                    return self.settle_withheld(winner, &pr, record).await;
8875                }
8876            }
8877        }
8878        let (title, body) =
8879            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8880        Ok(Some(PrMessage { title, body }))
8881    }
8882
8883    /// Describe the whole change from its final diff in either run mode.
8884    ///
8885    /// The summary is tied to the final commit and to the merge base of the
8886    /// resolved remote base; a record for any other pair is dropped and the
8887    /// agent asked again, so a description never outlives the diff it was
8888    /// written from. The seat is read-only and the prompt tells it to read the
8889    /// diff and the code around it. `[roles] synthesizer` selects the agents
8890    /// (unset: the default pick), each asked at most once, advancing on an
8891    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8892    /// carries the reason it is unavailable; nothing is invented then.
8893    async fn summarize_change(
8894        &mut self,
8895        winner: &Candidate,
8896        head: Option<&str>,
8897    ) -> std::result::Result<(), String> {
8898        let Some(head) = head else {
8899            return Err("the final commit could not be read".to_owned());
8900        };
8901        let remote = self.state.config.merge.remote.clone();
8902        let tracking = format!("{remote}/{}", self.state.base_branch);
8903        if !git::rev_exists(&winner.worktree, &tracking).await {
8904            return Err(format!("the remote base `{tracking}` could not be read"));
8905        }
8906        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8907            Ok(b) if !b.is_empty() => b,
8908            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8909        };
8910        let evidence = crate::pr_outcome::summary_evidence(
8911            &self.state,
8912            crate::pr_outcome::FinalHead::At(head),
8913        );
8914        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8915        if self
8916            .state
8917            .change_summary
8918            .as_ref()
8919            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8920        {
8921            return Ok(());
8922        }
8923        if self.state.change_summary.take().is_some() {
8924            self.state.event(
8925                "pr-summary",
8926                "the final diff or evidence changed; the earlier change summary was discarded",
8927            );
8928        }
8929        // Save the source snapshot even if no generation agent is available.
8930        if let Err(e) = write_artifact(
8931            &self.state,
8932            &format!("public-description-{fingerprint}.md"),
8933            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8934        ) {
8935            tracing::warn!("could not save public description evidence: {e:#}");
8936        }
8937        let chain = match agent::pick_chain(
8938            &self.state.config.agents,
8939            self.state.config.roles.synthesizer.as_ref(),
8940            &agent::installed,
8941            "synthesizer",
8942        ) {
8943            Ok(chain) => chain,
8944            Err(e) => {
8945                tracing::warn!("no agent for the change summary: {e:#}");
8946                self.state.event("pr-summary", "no summary agent available");
8947                return Err("no agent was available to summarize it".to_owned());
8948            }
8949        };
8950        let subjects = git::subjects(&winner.worktree, &base, head)
8951            .await
8952            .unwrap_or_default();
8953        let stat = git::diff_stat(&winner.worktree, &base, head)
8954            .await
8955            .unwrap_or_default();
8956        let stat: String = stat.chars().take(6_000).collect();
8957        let mut prompt = crate::prompt::change_summary_in(
8958            &base,
8959            head,
8960            &subjects,
8961            stat.trim(),
8962            &self.state.config.graph.github_language,
8963        );
8964        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8965        let sessions = self.state.config.graph.sessions;
8966        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8967        let prompts = self.state.config.prompts.clone();
8968        let cache = self.state.config.cache_dir();
8969        let run = self.state.id.clone();
8970        for (i, spec) in chain.into_iter().enumerate() {
8971            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8972            let github_spec = match agent::for_github_text(
8973                &spec,
8974                &self.state.config.graph.github_language,
8975                &winner.worktree,
8976            ) {
8977                Ok(spec) => spec,
8978                Err(e) => {
8979                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8980                    continue;
8981                }
8982            };
8983            let job = SeatJob {
8984                spec: github_spec,
8985                seat,
8986                cwd: winner.worktree.clone(),
8987                prompt: prompt.clone(),
8988                timeout,
8989                allow_write: false,
8990                sessions,
8991                artifacts: agent::artifacts_dir(&self.state.dir()),
8992                stem: if i == 0 {
8993                    "change-summary".to_owned()
8994                } else {
8995                    format!("change-summary-{}", spec.id)
8996                },
8997                handover: None,
8998            };
8999            let ctx = WaveCtx {
9000                run: &run,
9001                node: "pr-summary",
9002                prompts: &prompts,
9003                cache: cache.as_deref(),
9004                round: None,
9005                carry_seats: false,
9006                pauses: Some((&self.pause, &self.interrupt)),
9007            };
9008            let (seat, outcome) =
9009                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9010            self.state.seats.insert(seat.key.clone(), seat);
9011            let text = match outcome {
9012                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9013                AgentOutcome::ResourceBlocked(why) => {
9014                    // No agent was asked: the run stops resumably before
9015                    // anything is published (see `guarded_pr_message`).
9016                    if self.state.resource_block.is_none() {
9017                        self.state.resource_block = Some(crate::run::ResourceBlock {
9018                            cause: crate::run::ResourceBlockCause::Other,
9019                            node: "pr-summary".to_owned(),
9020                            kind: "shared resource".to_owned(),
9021                            detail: why,
9022                            at: Timestamp::now(),
9023                        });
9024                    }
9025                    return Err("a shared resource was busy".to_owned());
9026                }
9027                _ => continue,
9028            };
9029            let blind = &self.state.config.blind;
9030            let Some(title) = summary_title(&text) else {
9031                continue;
9032            };
9033            let body: String = summary_without_title(&text)
9034                .lines()
9035                .filter(|l| !l.contains("magi:run/"))
9036                .collect::<Vec<_>>()
9037                .join("\n");
9038            let body = blind::sanitize_prose(body.trim(), blind);
9039            let title = blind::sanitize_prose(&title, blind);
9040            if body.trim().is_empty() || title.trim().is_empty() {
9041                continue;
9042            }
9043            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9044                continue;
9045            }
9046            // The branch must still be the one the agent was shown.
9047            if git::rev_parse(&winner.worktree, "HEAD")
9048                .await
9049                .ok()
9050                .as_deref()
9051                != Some(head)
9052            {
9053                return Err("the branch moved while the summary was written".to_owned());
9054            }
9055            if git::merge_base(&winner.worktree, &tracking, head)
9056                .await
9057                .ok()
9058                .as_deref()
9059                != Some(base.as_str())
9060                || crate::pr_outcome::summary_evidence(
9061                    &self.state,
9062                    crate::pr_outcome::FinalHead::At(head),
9063                ) != evidence
9064            {
9065                return Err("the summary evidence changed while it was written".to_owned());
9066            }
9067            self.state.change_summary = Some(crate::run::ChangeSummary {
9068                head: head.to_owned(),
9069                base,
9070                evidence: fingerprint,
9071                title,
9072                body,
9073            });
9074            self.state.event(
9075                "pr-summary",
9076                format!("change summary written by {} for the final diff", spec.id),
9077            );
9078            if let Err(e) = self.state.save() {
9079                tracing::warn!("could not save the change summary: {e:#}");
9080            }
9081            return Ok(());
9082        }
9083        self.state
9084            .event("pr-summary", "no agent produced a change summary");
9085        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9086    }
9087
9088    /// One read-only rewrite request for a withheld description. The
9089    /// implementer's own conversation is tried first when it exists and can be
9090    /// resumed; then the `[roles] language_judge` chain (or the default pick)
9091    /// writes it in a separate conversation, skipping the implementer's agent.
9092    /// The prompt carries the whole text, so no agent needs any memory. Each
9093    /// agent is asked at most once and the chain advances on an error, quota,
9094    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
9095    /// that has run out of quota must not end the rewrite. Nothing is recorded
9096    /// as a handover.
9097    async fn rewrite_pr_text(
9098        &mut self,
9099        winner: &Candidate,
9100        pr: &PrMessage,
9101        violations: &[crate::github_text::Violation],
9102    ) -> Option<String> {
9103        let prompt = format!(
9104            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9105            pr.title, pr.body
9106        );
9107        let sessions = self.state.config.graph.sessions;
9108        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9109        // (spec, the resumed implementer's seat). A fallback seat is created
9110        // only when its turn comes, so an unused fallback leaves no record.
9111        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9112        let key = format!("impl-{}", winner.label);
9113        if let Some(spec) = self
9114            .state
9115            .config
9116            .agents
9117            .iter()
9118            .find(|a| a.id == winner.agent)
9119            .cloned()
9120            && let Some(existing) = self.state.seats.get(&key)
9121            && existing.agent == spec.id
9122            && agent::has_session(spec.kind, existing, sessions)
9123        {
9124            attempts.push((spec, Some(existing.clone())));
9125        }
9126        match agent::pick_chain(
9127            &self.state.config.agents,
9128            self.state.config.roles.language_judge.as_ref(),
9129            &agent::installed,
9130            "language_judge",
9131        ) {
9132            Ok(chain) => {
9133                for spec in chain {
9134                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9135                        continue;
9136                    }
9137                    attempts.push((spec, None));
9138                }
9139            }
9140            Err(e) if attempts.is_empty() => {
9141                tracing::warn!("no agent for the description rewrite: {e:#}");
9142                self.state
9143                    .event("github-text", "no rewrite agent available");
9144                return None;
9145            }
9146            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9147        }
9148        let prompts = self.state.config.prompts.clone();
9149        let cache = self.state.config.cache_dir();
9150        let run = self.state.id.clone();
9151        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9152            let resumed = resumed_seat.is_some();
9153            let seat = resumed_seat.unwrap_or_else(|| {
9154                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9155            });
9156            let github_spec = match agent::for_github_text(
9157                &spec,
9158                &self.state.config.graph.github_language,
9159                &winner.worktree,
9160            ) {
9161                Ok(spec) => spec,
9162                Err(e) => {
9163                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9164                    continue;
9165                }
9166            };
9167            let job = SeatJob {
9168                spec: github_spec,
9169                seat,
9170                cwd: winner.worktree.clone(),
9171                prompt: prompt.clone(),
9172                timeout: retry_budget(timeout_full, resumed),
9173                allow_write: false,
9174                sessions,
9175                artifacts: agent::artifacts_dir(&self.state.dir()),
9176                stem: if i == 0 {
9177                    "github-text-rewrite".to_owned()
9178                } else {
9179                    format!("github-text-rewrite-{}", spec.id)
9180                },
9181                handover: None,
9182            };
9183            let ctx = WaveCtx {
9184                run: &run,
9185                node: "github-text",
9186                prompts: &prompts,
9187                cache: cache.as_deref(),
9188                round: None,
9189                carry_seats: false,
9190                pauses: Some((&self.pause, &self.interrupt)),
9191            };
9192            let (seat, outcome) =
9193                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9194            self.state.seats.insert(seat.key.clone(), seat);
9195            match outcome {
9196                AgentOutcome::Ok(o)
9197                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9198                {
9199                    return Some(o.text);
9200                }
9201                AgentOutcome::ResourceBlocked(_) => return None,
9202                _ => {}
9203            }
9204        }
9205        None
9206    }
9207
9208    /// Abandon the standing posting-gate question, if any.
9209    fn retire_github_text_question(&mut self, why: &str) {
9210        let Some(id) = self
9211            .state
9212            .github_text
9213            .as_ref()
9214            .and_then(|g| g.question.clone())
9215        else {
9216            return;
9217        };
9218        let _ = ask::Questions::open().update(&id, |q| {
9219            q.abandon(why);
9220            Ok(())
9221        });
9222    }
9223
9224    /// Carry a withheld text to its end: file the question, park while it is
9225    /// open, and on an answer, silence or timeout pick the text to post. The
9226    /// decision is saved before it is used, so it applies once.
9227    async fn settle_withheld(
9228        &mut self,
9229        winner: &Candidate,
9230        pr: &PrMessage,
9231        mut g: crate::run::GithubTextAsk,
9232    ) -> Result<Option<PrMessage>> {
9233        use crate::github_text as gt;
9234        let store = ask::Questions::open();
9235        let timeout = self.state.config.graph.answer_timeout;
9236        if !g.resolved {
9237            let mut standing = g
9238                .question
9239                .as_ref()
9240                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9241            if let Some(q) = standing.as_ref()
9242                && q.status.open()
9243                && gt::expired(q, timeout)
9244            {
9245                let why = format!("no answer within {}s of asking", timeout.max(1));
9246                standing = store
9247                    .update(&q.id, |q| {
9248                        q.abandon(&why);
9249                        Ok(())
9250                    })
9251                    .ok()
9252                    .map(|(q, ())| q);
9253            }
9254            let mut ask_again: Option<Option<Vec<String>>> = None;
9255            match standing {
9256                // Recorded but never written (a stop between the two saves),
9257                // or removed: ask again without spending the retry.
9258                None if g.asks <= 1 => {
9259                    g.asks = 0;
9260                    ask_again = Some(None);
9261                }
9262                None => {
9263                    g.resolved = true;
9264                    g.chosen_title = None;
9265                    g.choice = Some(GithubTextChoice::Fallback);
9266                }
9267                Some(q) if q.status.open() => {
9268                    self.park_on_github_text(&g)?;
9269                    return Ok(None);
9270                }
9271                Some(q) => match gt::read_reply(&q) {
9272                    gt::Reply::Fallback => {
9273                        g.resolved = true;
9274                        g.chosen_title = None;
9275                        g.choice = Some(GithubTextChoice::Fallback);
9276                    }
9277                    gt::Reply::Original => {
9278                        g.resolved = true;
9279                        g.chosen_title = None;
9280                        g.choice = Some(GithubTextChoice::Original);
9281                    }
9282                    gt::Reply::Rewrite => {
9283                        g.resolved = true;
9284                        g.chosen_title = None;
9285                        g.choice = Some(if g.rewrite.is_some() {
9286                            GithubTextChoice::Rewrite
9287                        } else {
9288                            GithubTextChoice::Fallback
9289                        });
9290                    }
9291                    gt::Reply::Title(text) => {
9292                        let first = text
9293                            .lines()
9294                            .map(str::trim)
9295                            .find(|l| !l.is_empty())
9296                            .unwrap_or("");
9297                        let decision = if first.is_empty() {
9298                            None
9299                        } else {
9300                            let candidate = PrMessage {
9301                                title: first.to_owned(),
9302                                body: String::new(),
9303                            };
9304                            self.judge_pr_language(&winner.worktree, &candidate).await
9305                        };
9306                        match gt::vet_title(
9307                            &text,
9308                            decision,
9309                            self.state.config.graph.github_english(),
9310                        ) {
9311                            Ok(title) => {
9312                                g.resolved = true;
9313                                g.chosen_title = Some(title);
9314                                g.choice = Some(GithubTextChoice::MyTitle);
9315                            }
9316                            Err(cats) if g.asks < 2 => {
9317                                ask_again = Some(Some(
9318                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9319                                ))
9320                            }
9321                            Err(_) => {
9322                                g.resolved = true;
9323                                g.chosen_title = None;
9324                                g.choice = Some(GithubTextChoice::Fallback);
9325                            }
9326                        }
9327                    }
9328                },
9329            }
9330            if let Some(rejected) = ask_again {
9331                // A retry (after a rejected replacement title) is still about
9332                // the original text: the same ORIGINAL / FALLBACK comparison.
9333                let w = gt::Withheld {
9334                    title: g.title,
9335                    body: g.body,
9336                    categories: g.categories.clone(),
9337                };
9338                // Named by the original text's fingerprint alone, so a resume
9339                // rewrites the same file and the detail stays identical.
9340                let artifact = match crate::run::write_artifact(
9341                    &self.state,
9342                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
9343                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9344                ) {
9345                    Ok(p) => Some(p),
9346                    Err(e) => {
9347                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
9348                        None
9349                    }
9350                };
9351                let (detail, comparison) = gt::question_content(
9352                    &w,
9353                    &pr.title,
9354                    &pr.body,
9355                    &self.state.id,
9356                    rejected.as_deref(),
9357                    artifact.as_deref(),
9358                    g.rewrite.as_ref(),
9359                );
9360                let mut q = ask::Question::new(
9361                    self.state.id.clone(),
9362                    gt::ASK_NODE.to_owned(),
9363                    gt::ASK_SEAT.to_owned(),
9364                    gt::question_summary(&self.state.config.graph.language, &w),
9365                    detail,
9366                    gt::question_choices(&w, g.rewrite.as_ref()),
9367                );
9368                q.posting_comparison = Some(comparison);
9369                q.answer_timeout = timeout;
9370                // Recorded before it is written: a stop in between leaves a
9371                // record naming a question the resume then files again.
9372                g.question = Some(q.id.clone());
9373                g.asks += 1;
9374                self.state.github_text = Some(g.clone());
9375                self.state.event(
9376                    "github-text",
9377                    format!("asking the owner about the withheld text ({})", q.short()),
9378                );
9379                self.state.save()?;
9380                store
9381                    .put_executable(&mut q)
9382                    .context("file the posting-gate question")?;
9383                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9384                    tracing::warn!(
9385                        "could not notify about posting-gate question {}: {e:#}",
9386                        q.short()
9387                    );
9388                }
9389                self.park_on_github_text(&g)?;
9390                return Ok(None);
9391            }
9392        }
9393        let resolved = g.chosen_title.clone();
9394        let original = g.choice == Some(GithubTextChoice::Original);
9395        // Field names only: the event log never records the text.
9396        self.state.event(
9397            "github-text",
9398            if original {
9399                format!(
9400                    "posting the original text for: {}",
9401                    match (g.title, g.body) {
9402                        (true, true) => "title, description",
9403                        (true, false) => "title",
9404                        _ => "description",
9405                    }
9406                )
9407            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9408                "posting the saved rewrite title and description".to_owned()
9409            } else if resolved.is_some() {
9410                "posting the owner's replacement title".to_owned()
9411            } else {
9412                "posting the neutral text for the withheld fields".to_owned()
9413            },
9414        );
9415        self.state.github_text = Some(g.clone());
9416        self.state.save()?;
9417        if g.choice == Some(GithubTextChoice::Rewrite)
9418            && let Some(r) = g.rewrite.as_ref()
9419        {
9420            return Ok(Some(PrMessage {
9421                title: r.title.clone(),
9422                body: r.body.clone(),
9423            }));
9424        }
9425        let w = gt::Withheld {
9426            title: g.title,
9427            body: g.body,
9428            categories: g.categories.clone(),
9429        };
9430        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9431        if let Some(t) = resolved {
9432            title = t;
9433        }
9434        // Every withheld field is now neutral or vetted English.
9435        let english = gt::LanguageDecision {
9436            title_english: true,
9437            body_english: true,
9438            body_complete: true,
9439        };
9440        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9441        if original {
9442            // The owner's explicit choice overrides the gate for the fields it
9443            // was made for; the others keep what the gate produced.
9444            if g.title {
9445                title = pr.title.clone();
9446            }
9447            if g.body {
9448                body = pr.body.clone();
9449            }
9450        }
9451        Ok(Some(PrMessage { title, body }))
9452    }
9453
9454    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9455        self.state.github_text = Some(g.clone());
9456        self.state.end_pass_parked(ParkCause::Question);
9457        self.state.parked = true;
9458        self.state.event(
9459            "github-text",
9460            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9461        );
9462        self.state.save()
9463    }
9464
9465    /// Ask `[roles] language_judge` about this text and record which source
9466    /// decided; the text itself is never recorded.
9467    async fn judge_pr_language(
9468        &mut self,
9469        cwd: &Path,
9470        pr: &PrMessage,
9471    ) -> Option<crate::github_text::LanguageDecision> {
9472        self.state.config.roles.language_judge.as_ref()?;
9473        if !self.state.config.graph.language_gate() {
9474            return None;
9475        }
9476        let decision =
9477            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9478        self.state.event(
9479            "github-text",
9480            if decision.is_some() {
9481                "language decided by the language judge"
9482            } else {
9483                "language judge unavailable; using built-in heuristics"
9484            },
9485        );
9486        decision
9487    }
9488
9489    async fn merge(&mut self) -> Result<()> {
9490        if self.resource_halt()? || self.hold_base_sync().await? {
9491            return Ok(());
9492        }
9493        // Same reasoning as `gate`: ask the review and gate records directly
9494        // rather than `status`, which a solo-candidate `judge`/`deliberate`
9495        // skip can rewrite on reentry to something that no longer says
9496        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
9497        // so a hand-off (open findings, green verification) reaches merge
9498        // exactly like a genuinely clean round does.
9499        //
9500        // A run resumed mid-`land` never reaches here at all: `execute`
9501        // recognises `RunStatus::Landing` before it even calls `prep`, and
9502        // routes straight to `run_land` instead. That has to happen a level
9503        // up from this function, not with a check in here, because
9504        // `review_loop`'s own status recomputation (see its doc) runs
9505        // *before* `merge` on every reentry and would otherwise overwrite
9506        // the `Landing` marker with `Gating` before this node ever saw it.
9507        if self
9508            .state
9509            .base_sync
9510            .as_ref()
9511            .is_some_and(|s| s.conflict.is_some())
9512            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9513                != Some(RunStatus::Gating)
9514            // `gate_ran == false` is not "passed" - `gate` leaves it false
9515            // both before it has ever run and when its last attempt was
9516            // resource-blocked (see `Runner::gate`'s own doc), and neither is
9517            // permission to merge on nothing but the review record. Only a
9518            // gate that actually ran may proceed; an intentional local
9519            // review can also finish with verification explicitly not configured.
9520            // `RunState::gate_status` computes that distinction.
9521            || !self.state.gate_allows_progress()
9522        {
9523            return Ok(());
9524        }
9525        // This node's own record, not `status`: `status == Ready` is not
9526        // unique to the harmless `MergeMode::None` path this line was
9527        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
9528        // run's PR was closed without merging — and on that run `mode` is
9529        // still `Pr`, so a reentry that fell through here would push and
9530        // open a second pull request. `self.state.merge` is set exactly once
9531        // this node (or `land`) has already produced a verdict, under every
9532        // mode, which is what "already done" actually means here.
9533        if self.state.merge.is_some() && !self.state.pr_publication_pending {
9534            return Ok(());
9535        }
9536        let Some(winner) = self.state.winner().cloned() else {
9537            return Ok(());
9538        };
9539        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9540        if self.state.gate_head.as_deref() != Some(head.as_str()) {
9541            self.state.status = RunStatus::Blocked;
9542            self.state.event(
9543                "gate",
9544                "winner changed after gate; fresh verification required",
9545            );
9546            self.state.save()?;
9547            return Ok(());
9548        }
9549        let wait = self.repository_wait();
9550        if !verify_current_head(&mut self.state, &winner, &wait).await? {
9551            return Ok(());
9552        }
9553        let repo = self.state.repo.clone();
9554        let base = self.state.base_branch.clone();
9555        let mode = self.state.config.merge.mode;
9556        let style = self.state.config.merge.style;
9557        let facts = if is_review_run(&self.state) {
9558            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9559            let start = review_base(
9560                &repo,
9561                &self.state.config.merge.remote,
9562                &base,
9563                &self.state.base_commit,
9564                &winner.branch,
9565            )
9566            .await;
9567            branch_facts(&repo, &start, &winner.branch).await
9568        } else {
9569            None
9570        };
9571        // `None` when the base could not be freshly read: then an adopted
9572        // pull request's title is left alone.
9573        let leaked = if is_review_run(&self.state) {
9574            leaked_subjects(&self.state, &winner.branch).await
9575        } else {
9576            Some(Vec::new())
9577        };
9578        // `None`: parked on the owner's word about a withheld title or body.
9579        // Nothing has been pushed or opened, and `merge` stays unrecorded so
9580        // the resume arrives here again.
9581        let Some(pr) = self
9582            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9583            .await?
9584        else {
9585            return Ok(());
9586        };
9587        let message = pr.commit_message();
9588
9589        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9590        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9591            return Ok(());
9592        }
9593        let outcome = match mode {
9594            MergeMode::None => MergeOutcome {
9595                mode,
9596                ok: true,
9597                detail: manual_merge_command(style, &repo, &winner.branch, &message),
9598                empty: false,
9599            },
9600            MergeMode::Pr | MergeMode::Local
9601                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9602            {
9603                MergeOutcome {
9604                    mode,
9605                    ok: false,
9606                    detail: empty_candidate_detail(&self.state, &base),
9607                    empty: true,
9608                }
9609            }
9610            MergeMode::Local => {
9611                let on = git::current_branch(&repo).await?;
9612                if on.as_deref() != Some(base.as_str()) {
9613                    MergeOutcome {
9614                        mode,
9615                        ok: false,
9616                        detail: format!(
9617                            "{} has {} checked out, not the base branch {base}",
9618                            repo.display(),
9619                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
9620                        ),
9621                        empty: false,
9622                    }
9623                } else if !git::is_clean(&repo).await? {
9624                    MergeOutcome {
9625                        mode,
9626                        ok: false,
9627                        detail: format!("{} is dirty; refusing to merge", repo.display()),
9628                        empty: false,
9629                    }
9630                } else {
9631                    let out = match style {
9632                        MergeStyle::Merge => {
9633                            git::merge_no_ff(&repo, &winner.branch, &message).await?
9634                        }
9635                        MergeStyle::Squash => {
9636                            git::merge_squash(&repo, &winner.branch, &message).await?
9637                        }
9638                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9639                    };
9640                    MergeOutcome {
9641                        mode,
9642                        ok: out.ok(),
9643                        detail: if out.ok() { out.stdout } else { out.stderr },
9644                        empty: false,
9645                    }
9646                }
9647            }
9648            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9649                self.state.pr_publication_pending = true;
9650                self.state.save()?;
9651                let out =
9652                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9653                self.state.pr_publication_pending = out.is_err();
9654                MergeOutcome {
9655                    mode,
9656                    ok: out.is_ok(),
9657                    detail: match out {
9658                        Ok(url) => url,
9659                        Err(e) => format!("{e:#}"),
9660                    },
9661                    empty: false,
9662                }
9663            }
9664            MergeMode::Pr => {
9665                let remote = self.state.config.merge.remote.clone();
9666                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9667                let pushed = if self.state.foreign_checkout.is_some() {
9668                    let refspec = format!("{delivery_head}:refs/heads/{public}");
9669                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9670                } else {
9671                    git::push(&winner.worktree, &remote, &winner.branch).await?
9672                };
9673                if !pushed.ok() {
9674                    MergeOutcome {
9675                        mode,
9676                        ok: false,
9677                        detail: pushed.stderr,
9678                        empty: false,
9679                    }
9680                } else {
9681                    if self.state.foreign_checkout.is_some() {
9682                        self.state.pr_pushed_head = Some(delivery_head.clone());
9683                    }
9684                    // A retry or resume of a run whose branch already has an
9685                    // open pull request adopts it rather than failing on a
9686                    // duplicate. Only this winner branch into this base:
9687                    // `branch_for` derives the name from the run id, so a
9688                    // different run's pull request never matches.
9689                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9690                    let out = match pr_merge_plan(found) {
9691                        PrPlan::Create => {
9692                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9693                                .await
9694                        }
9695                        PrPlan::Adopt { url, title } => {
9696                            self.state
9697                                .event("merge", format!("Pr: adopted open pull request {url}"));
9698                            if self.state.explicit_github_text_selection() {
9699                                // An explicit publication selection applies the prepared
9700                                // payload; acceptance alone is not publication proof.
9701                                land::set_pr_text_verbatim(
9702                                    &winner.worktree,
9703                                    &url,
9704                                    &pr.title,
9705                                    &pr.body,
9706                                )
9707                                .await
9708                                .map(|()| url)
9709                            } else {
9710                                if title != pr.title
9711                                    && (!is_review_run(&self.state)
9712                                        || leaked
9713                                            .as_deref()
9714                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9715                                    && let Err(e) =
9716                                        if self.state.github_text.as_ref().is_some_and(|g| {
9717                                            g.resolved
9718                                                && g.title
9719                                                && g.choice == Some(GithubTextChoice::Original)
9720                                        }) {
9721                                            // The owner chose to post this title unmodified.
9722                                            land::set_pr_title_verbatim(
9723                                                &winner.worktree,
9724                                                &url,
9725                                                &pr.title,
9726                                            )
9727                                            .await
9728                                        } else {
9729                                            land::set_pr_title(
9730                                                &mut self.state,
9731                                                &winner.worktree,
9732                                                &url,
9733                                                &pr.title,
9734                                            )
9735                                            .await
9736                                        }
9737                                {
9738                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9739                                    self.state
9740                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9741                                }
9742                                Ok(url)
9743                            }
9744                        }
9745                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9746                    };
9747                    match out {
9748                        Ok(url) => MergeOutcome {
9749                            mode,
9750                            ok: true,
9751                            detail: url,
9752                            empty: false,
9753                        },
9754                        Err(e) => MergeOutcome {
9755                            mode,
9756                            ok: false,
9757                            detail: e.to_string(),
9758                            empty: false,
9759                        },
9760                    }
9761                }
9762            }
9763        };
9764
9765        self.state.status = match (mode, outcome.ok) {
9766            (MergeMode::None, _) => RunStatus::Ready,
9767            (_, true) if outcome.empty => RunStatus::Ready,
9768            (MergeMode::Pr, true) => RunStatus::Ready,
9769            (_, true) => RunStatus::Merged,
9770            (_, false) => RunStatus::Blocked,
9771        };
9772        self.state.event(
9773            "merge",
9774            format!(
9775                "{:?}: {}",
9776                mode,
9777                outcome.detail.lines().next().unwrap_or("")
9778            ),
9779        );
9780        if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9781            self.state.record_github_text_publication(&outcome.detail);
9782        }
9783        self.state.merge = Some(outcome);
9784        self.state.save()?;
9785
9786        // The PR is open and the run would historically stop here, leaving the
9787        // operator to watch checks, feed review comments back to a fixer, and
9788        // merge. That was done by hand six times in one session before this
9789        // existed. Opt-in, because merging is the one irreversible thing magi
9790        // can do to a repository.
9791        if self.state.config.graph.land
9792            && mode == MergeMode::Pr
9793            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9794        {
9795            self.run_land().await?;
9796        }
9797        // `run_land` may have left `status` at `Landing` - still waiting on
9798        // CI or the owner's approval, not actually settled - so this has to
9799        // read whatever `status` ended up as here, not the `Merged` this
9800        // function set a few lines up.
9801        self.settle_questions();
9802        Ok(())
9803    }
9804
9805    /// Enter `land`.
9806    ///
9807    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9808    /// a resumed run's re-entry. `land::land` itself is what serialises the
9809    /// two git-mutating moments inside the loop — the rebase push and
9810    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9811    /// to hold a lock across the whole call, and doing so would serialise
9812    /// this run's CI wait against a *different* run's land-approval resume
9813    /// in the same repository, which is exactly the "must not wait on
9814    /// another task" property the daemon's slot-freeing exists to give.
9815    async fn run_land(&mut self) -> Result<()> {
9816        crate::pr_head::ensure_owner(&self.state)?;
9817        let url = self
9818            .state
9819            .pr
9820            .as_ref()
9821            .map(|pr| pr.url.clone())
9822            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9823            .unwrap_or_default();
9824        let url = url.lines().next().unwrap_or("").trim().to_owned();
9825        if !url.starts_with("http") {
9826            return Ok(());
9827        }
9828        // A land failure is not a lost run: the work is on a branch and the
9829        // pull request is open, which is exactly where a human takes over.
9830        loop {
9831            match land::land(&mut self.state, &url).await {
9832                Ok(_)
9833                    if self
9834                        .state
9835                        .merge_repairs
9836                        .last()
9837                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9838                {
9839                    if self.repair_merge_review().await? {
9840                        continue;
9841                    }
9842                }
9843                Ok(_) if self.state.resource_block.is_some() => {
9844                    return self.stop_for_resource();
9845                }
9846                Ok(pr) if self.state.parked => {
9847                    // `land` already saved the parked marker; nothing here
9848                    // overrides `status` back to a terminal value while an
9849                    // approval is still outstanding.
9850                    let _ = pr;
9851                }
9852                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9853                    self.state.status = RunStatus::Superseded;
9854                    self.state.save()?;
9855                }
9856                Ok(pr) => {
9857                    self.state.status = match pr.state {
9858                        land::PrLifecycle::Merged => RunStatus::Merged,
9859                        _ => RunStatus::Blocked,
9860                    };
9861                    // Downstream of a confirmed merge only - see
9862                    // `bump::should_release_bump`'s own doc for why this one
9863                    // check covers all three of `land`'s success paths.
9864                    // Best-effort: the run already landed, so a failure here
9865                    // (the decision call, `gh`, `cargo`) is recorded and never
9866                    // turns a landed run into a failed one.
9867                    if bump::should_release_bump(self.state.status)
9868                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9869                    {
9870                        // The event is the run's own record. Not-eligible cases
9871                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9872                        // `Err` is a bump that was tried and failed:
9873                        // `after_merge` itself raises the operator notice for
9874                        // that, whether or not a release PR exists yet.
9875                        self.state
9876                            .event("bump", format!("release bump skipped: {e:#}"));
9877                    }
9878                    // Independent of the bump, and best-effort in the same way:
9879                    // findings the merge left open become follow-up tasks.
9880                    if self.state.status == RunStatus::Merged {
9881                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9882                    }
9883                    self.state.save()?;
9884                }
9885                Err(e) => {
9886                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9887                        return Err(e);
9888                    }
9889                    self.state.status = RunStatus::Blocked;
9890                    self.state.event("land", format!("gave up: {e}"));
9891                    self.state.save()?;
9892                }
9893            }
9894            break;
9895        }
9896        Ok(())
9897    }
9898
9899    /// One repair cycle stays under the existing driver/task claim. An
9900    /// interrupted fixer is rolled back and held rather than run twice.
9901    async fn repair_merge_review(&mut self) -> Result<bool> {
9902        use crate::run::MergeRepairPhase as Phase;
9903        crate::pr_head::ensure_owner(&self.state)?;
9904        let i = self.state.merge_repairs.len() - 1;
9905        let request = self.state.merge_repairs[i].clone();
9906        if request.phase == Phase::Fixing {
9907            let restored = match self.state.winner().cloned() {
9908                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9909                None => Err(anyhow::anyhow!("repair has no winning branch")),
9910            };
9911            let why = match restored {
9912                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9913                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9914                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9915            };
9916            self.hold_merge_repair(i, why)?;
9917            return Ok(false);
9918        }
9919        // A completed repair gate is a publication checkpoint. On reentry,
9920        // reject external changes before the ordinary gate path can replace
9921        // that evidence by verifying a new head. Requested repairs still
9922        // start their own review generation below.
9923        if request.phase == Phase::Reviewing && self.state.gate_ran {
9924            let winner = self
9925                .state
9926                .winner()
9927                .context("repair has no winning branch")?;
9928            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9929            if self.state.gate_head.as_deref() != Some(head.as_str())
9930                || !git::is_clean(&winner.worktree).await?
9931            {
9932                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9933                return Ok(false);
9934            }
9935        }
9936        let failure = if i >= self.state.config.graph.land_rounds
9937            || self.state.config.graph.review_rounds == 0
9938        {
9939            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9940        } else {
9941            None
9942        };
9943        if let Some(why) = failure {
9944            self.hold_merge_repair(i, why)?;
9945            return Ok(false);
9946        }
9947        if request.phase == Phase::Requested {
9948            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9949                Ok(pr) => pr,
9950                Err(e) => {
9951                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9952                    return Ok(false);
9953                }
9954            };
9955            self.state.merge_repairs[i].phase = Phase::Fixing;
9956            self.state.land_approval = None;
9957            self.state.land_armed_head = None;
9958            self.state.save()?;
9959            // Serialize the exact records, including finding ids/details,
9960            // reconsideration reasons, final votes and verification output.
9961            let evidence = serde_json::to_string_pretty(&request)?;
9962            let reason = format!(
9963                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9964            );
9965            let budget = self.state.config.graph.land_rounds;
9966            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9967            match result {
9968                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9969                Ok(land::Fixed::Deferred) => {
9970                    // An unavailable fixer did not spend this repair request.
9971                    // Restore partial fallback work before retrying it, as for
9972                    // an interrupted repair, without closing the request.
9973                    let worktree = self
9974                        .state
9975                        .winner()
9976                        .context("repair has no winning branch")?
9977                        .worktree
9978                        .clone();
9979                    self.graft_restore(&worktree, &request.head).await?;
9980                    self.state.merge_repairs[i].phase = Phase::Requested;
9981                    self.stop_for_resource()?;
9982                    return Ok(false);
9983                }
9984                Ok(land::Fixed::Failed(why)) => {
9985                    self.hold_merge_repair(
9986                        i,
9987                        format!(
9988                            "repair failed: {why}; inspect the branch and run an explicit review"
9989                        ),
9990                    )?;
9991                    return Ok(false);
9992                }
9993                Err(e) => {
9994                    self.hold_merge_repair(
9995                        i,
9996                        format!(
9997                            "repair failed: {e:#}; inspect the branch and run an explicit review"
9998                        ),
9999                    )?;
10000                    return Ok(false);
10001                }
10002            }
10003            // Even no change gets a real new reviewer wave. The previous
10004            // evidence remains in the request; a former clean/concluded round
10005            // cannot short-circuit the new review or gate.
10006            self.begin_merge_repair_review(i)?;
10007        }
10008        self.review_loop().await?;
10009        self.gate().await?;
10010        if self.resource_halt()? || self.state.parked {
10011            return Ok(false);
10012        }
10013        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10014            != Some(RunStatus::Gating)
10015            || !self.state.gate_allows_progress()
10016        {
10017            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10018            return Ok(false);
10019        }
10020        // Review and gate may have added commits after the initial repair.
10021        // Publish exactly their final head to the existing PR, with a lease.
10022        let winner = self
10023            .state
10024            .winner()
10025            .cloned()
10026            .context("repair has no winning branch")?;
10027        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10028        if self.state.gate_head.as_deref() != Some(after.as_str())
10029            || !git::is_clean(&winner.worktree).await?
10030        {
10031            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10032            return Ok(false);
10033        }
10034        let expected = self
10035            .state
10036            .pr_pushed_head
10037            .as_deref()
10038            .unwrap_or(&request.head)
10039            .to_owned();
10040        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10041        // Push the checked commit itself: a local ref moving after the check
10042        // must not change which commit the forge receives.
10043        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10044            return Ok(false);
10045        }
10046        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10047        let refspec = format!("{after}:refs/heads/{public}");
10048        let push = git::git_raw(
10049            &winner.worktree,
10050            &["push", &lease, &self.state.config.merge.remote, &refspec],
10051        )
10052        .await;
10053        match push {
10054            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10055            Ok(out) => {
10056                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10057                return Ok(false);
10058            }
10059            Err(e) => {
10060                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10061                return Ok(false);
10062            }
10063        }
10064        self.state.merge_repairs[i].phase = Phase::Complete;
10065        self.state.status = RunStatus::Landing;
10066        self.state.event(
10067            "repair",
10068            "new review and gate completed; fresh owner merge decision required",
10069        );
10070        self.state.save()?;
10071        Ok(true)
10072    }
10073
10074    fn merge_repair_artifacts(&self) -> PathBuf {
10075        let artifacts = agent::artifacts_dir(&self.state.dir());
10076        match self
10077            .state
10078            .merge_repairs
10079            .last()
10080            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10081        {
10082            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10083            None => artifacts,
10084        }
10085    }
10086
10087    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10088        if let Some(e2e) = self.state.head_e2e.take() {
10089            self.state
10090                .verification_history
10091                .push(crate::run::VerificationEvidence {
10092                    superseded_by: None,
10093                    archived_at: Some(Timestamp::now()),
10094                    fixes_spent: Some(self.state.gate_fixes.len()),
10095                    gate_head: None,
10096                    gate_ran: false,
10097                    gate: Vec::new(),
10098                    e2e: Some(e2e),
10099                });
10100        }
10101        self.state.reviews.clear();
10102        self.state.gate.clear();
10103        self.state.gate_ran = false;
10104        self.state.gate_head = None;
10105        self.state.gate_fixes.clear();
10106        self.state.gate_repair_e2e = None;
10107        self.state.pre_gate.clear();
10108        self.state.pre_gate_commit = None;
10109        self.state.contested_handoff = None;
10110        self.state.land_reruns = None;
10111        self.state.land_approval = None;
10112        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10113        self.state.status = RunStatus::Reviewing;
10114        self.state.save()
10115    }
10116
10117    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10118        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10119        self.state.merge_repairs[index].failure = Some(why.clone());
10120        self.state.status = RunStatus::Blocked;
10121        self.state.event("repair", why);
10122        self.state.save()
10123    }
10124
10125    // -------------------------------------------------------------- helpers
10126
10127    /// Fetch or create a seat, keeping its conversation across nodes.
10128    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10129        if let Some(existing) = self.state.seats.get(key)
10130            && existing.agent == agent
10131        {
10132            return existing.clone();
10133        }
10134        // A seat that changes agent mints its session id from the agent too,
10135        // like a handover: the old agent's uuid is already taken by the CLI.
10136        let fresh = if self.state.seats.contains_key(key) {
10137            handover_seat(key, agent, self.state.next_seat_seed())
10138        } else {
10139            SeatState::new(key, agent, self.state.seed)
10140        };
10141        self.state.seats.insert(key.to_owned(), fresh.clone());
10142        fresh
10143    }
10144
10145    /// The agent now holding seat `key`: `spec`, unless a handover moved the
10146    /// seat to another roster agent, in which case that agent. Nodes that
10147    /// continue a seat's conversation (deliberation, the votes, a reviewer's
10148    /// reconsideration) must keep talking to whoever answered it, not slip
10149    /// back to the agent that failed it.
10150    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10151        match self.state.seats.get(key) {
10152            Some(s) if s.agent != spec.id => {
10153                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10154            }
10155            _ => spec,
10156        }
10157    }
10158
10159    /// A candidate rendered for judging, with the leak policy applied.
10160    fn view(&self, c: &Candidate) -> CandidateView {
10161        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10162            .unwrap_or_default();
10163        let (patch, _) = blind::sanitize_patch(
10164            &format!("candidate {} patch", c.label),
10165            &raw,
10166            &self.state.config.blind,
10167        );
10168        CandidateView {
10169            label: c.label,
10170            branch: c.branch.clone(),
10171            summary: c.summary.clone(),
10172            stat: c.stat.clone(),
10173            patch,
10174        }
10175    }
10176
10177    /// The full candidate set as prompt text, for seats with no live session.
10178    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10179        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10180        prompt::judge(
10181            "(see above)",
10182            &views,
10183            self.roles.judges.len(),
10184            base_short,
10185            "en",
10186        )
10187    }
10188
10189    /// The final-vote prompt with everything a seat that has no session of its
10190    /// own needs: the candidates, the seat's own ranking and reasons, and the
10191    /// anonymised deliberation it took part in (only when there was one, so a
10192    /// handed-over seat never sees more than the seat it replaces did). The
10193    /// `Final vote` heading stays first.
10194    fn vote_prompt_full(
10195        &self,
10196        j: usize,
10197        labels: &[char],
10198        language: &str,
10199        candidates: &[Candidate],
10200        base_short: &str,
10201    ) -> String {
10202        let mut text = format!(
10203            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10204            prompt::final_vote(labels, language),
10205            self.state.instruction,
10206            self.candidate_block(candidates, base_short)
10207        );
10208        if let Some(own) = self
10209            .state
10210            .judgements
10211            .get(j)
10212            .filter(|r| !r.ranking.is_empty())
10213        {
10214            let reasons = own
10215                .reasons
10216                .iter()
10217                .map(|(k, v)| format!("- {k}: {v}"))
10218                .collect::<Vec<_>>()
10219                .join("\n");
10220            text.push_str(&format!(
10221                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10222                own.ranking.iter().collect::<String>(),
10223                if reasons.is_empty() {
10224                    ""
10225                } else {
10226                    ", because:\n"
10227                }
10228            ));
10229        }
10230        if !self.state.deliberation.is_empty() {
10231            text.push_str("\n# What was argued before this vote\n");
10232            for t in self.transcript(&[], j) {
10233                text.push_str(&format!(
10234                    "\n## {}{}\n\n{}\n",
10235                    t.who,
10236                    if t.is_self { " (you)" } else { "" },
10237                    t.body.trim()
10238                ));
10239            }
10240        }
10241        text
10242    }
10243
10244    /// Anonymised transcript for judge `self_idx`.
10245    ///
10246    /// The initial rankings are always the opening statements. Seeding them
10247    /// only when no turn had been taken yet meant every judge after the first
10248    /// argued against a single voice instead of against the actual split — the
10249    /// disagreement is the information, so it is always on the table.
10250    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10251        let mut turns = Vec::new();
10252        for j in &self.state.judgements {
10253            if j.ranking.is_empty() {
10254                continue;
10255            }
10256            let reasons = j
10257                .reasons
10258                .iter()
10259                .map(|(k, v)| format!("- {k}: {v}"))
10260                .collect::<Vec<_>>()
10261                .join("\n");
10262            turns.push(Turn {
10263                who: format!("Judge {} (opening ranking)", j.judge),
10264                is_self: j.judge == self_idx + 1,
10265                body: format!(
10266                    "Ranked {}{}{reasons}",
10267                    j.ranking.iter().collect::<String>(),
10268                    if reasons.is_empty() {
10269                        ""
10270                    } else {
10271                        ", because:\n"
10272                    }
10273                ),
10274            });
10275        }
10276        for t in self
10277            .state
10278            .deliberation
10279            .iter()
10280            .flat_map(|r| r.turns.iter())
10281            .chain(current)
10282        {
10283            turns.push(Turn {
10284                who: format!("Judge {}", t.judge),
10285                is_self: t.judge == self_idx + 1,
10286                body: t.body.clone(),
10287            });
10288        }
10289        turns
10290    }
10291}
10292
10293/// Does this seat still hold the context a follow-up prompt would rely on?
10294fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10295    agent::has_session(spec.kind, seat, sessions)
10296}
10297
10298/// The next entry in `roster` after `start`, never wrapping back to the
10299/// front, whose id is not in `tried` yet.
10300///
10301/// Starts one past `start` rather than at the front of `roster`: `start` is
10302/// the seat's own original position, and a seat whose candidate slot already
10303/// sits on the roster's second entry must fall through to the third next, not
10304/// restart at the first — which is very likely a different candidate's own
10305/// agent already. Never wraps back past `start`, for the same reason: an
10306/// entry earlier in the roster than the seat's own position is almost
10307/// certainly some *other* candidate slot's own agent, and once the tail of
10308/// the roster is exhausted there are no more untried agents for *this* seat
10309/// to fall through to — the caller's fallback chain ends there, exactly as
10310/// "no further untried agents remain in the list for that seat" asks for.
10311///
10312/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
10313/// the same id twice (an operator's `roles.implementers` typo, or a
10314/// `[[agents]]` list reused across roles) must not let
10315/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
10316/// over `roster` either finds an untried id or runs out, so this always
10317/// terminates regardless of duplicates.
10318fn next_untried_in_roster<'a>(
10319    roster: &'a [AgentSpec],
10320    start: usize,
10321    tried: &BTreeSet<String>,
10322) -> Option<&'a AgentSpec> {
10323    roster
10324        .get(start + 1..)?
10325        .iter()
10326        .find(|s| !tried.contains(&s.id))
10327}
10328
10329/// The successor for a seat, shared by all four seat kinds (implement, judge,
10330/// review, advise). `others` holds the ids that *currently* occupy the other
10331/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
10332/// sees them), so roster entries beyond the seat count act as spares: a failed
10333/// seat goes to an agent no other seat holds whenever the roster permits.
10334///
10335/// `carried` is `Some` only for the review loop's carried failure history.
10336/// Order: (1) forward from `start`, never wrapping, untried, not a carried
10337/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
10338/// whole roster: untried and not another seat's occupant; (3) the plain walk
10339/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
10340/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
10341/// a better outcome than an empty seat, and it keeps the answer to "is there
10342/// a successor at all" exactly what it was before occupants were considered,
10343/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
10344/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
10345/// carried failure is a cheaper bet than doubling an agent on the panel, and
10346/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
10347/// the same round are handled one at a time, so a seat later in the batch
10348/// sees an earlier one's new occupant but not yet its own freed agent.
10349fn pick_successor<'a>(
10350    roster: &'a [AgentSpec],
10351    start: usize,
10352    tried: &BTreeSet<String>,
10353    carried: Option<&BTreeSet<String>>,
10354    others: &BTreeSet<String>,
10355) -> Option<&'a AgentSpec> {
10356    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10357    roster
10358        .get(start + 1..)
10359        .and_then(|tail| {
10360            tail.iter()
10361                .filter(free)
10362                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10363        })
10364        .or_else(|| {
10365            carried
10366                .is_some()
10367                .then(|| roster.iter().find(free))
10368                .flatten()
10369        })
10370        .or_else(|| match carried {
10371            Some(c) => next_for_seat(roster, start, tried, c),
10372            None => next_untried_in_roster(roster, start, tried),
10373        })
10374}
10375
10376/// The next agent for a seat that carries its failure history across rounds
10377/// (the review loop). `round_tried` is this round's own bound and starts
10378/// empty every round; `carried_failed` only decides priority.
10379///
10380/// First: an id walking forward from `start`, never wrapping, that is neither
10381/// tried this round nor failed in an earlier one. Only when that is exhausted
10382/// does it rescue: the first roster id (in roster order, so this one *does*
10383/// look before `start`) not yet tried this round, which is by then a carried
10384/// failure. Each id is rescued at most once per round, so it cannot loop.
10385fn next_for_seat<'a>(
10386    roster: &'a [AgentSpec],
10387    start: usize,
10388    round_tried: &BTreeSet<String>,
10389    carried_failed: &BTreeSet<String>,
10390) -> Option<&'a AgentSpec> {
10391    roster
10392        .get(start + 1..)?
10393        .iter()
10394        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10395        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10396}
10397
10398/// Where a reviewer seat starts a round: the agent that last answered it when
10399/// it is still on the roster and not marked failed, else the spec's own agent
10400/// unless it failed, else the next roster agent that has not failed, else the
10401/// spec's own agent again (the whole roster failed). Ids no longer on the
10402/// roster are ignored. An empty roster has no handover, so the spec stands.
10403fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10404    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10405        return spec;
10406    };
10407    let ok = |id: &str| !h.failed.contains(id);
10408    if let Some(last) = h.last_ok.as_deref()
10409        && ok(last)
10410        && let Some(s) = roster.iter().find(|s| s.id == last)
10411    {
10412        return s.clone();
10413    }
10414    if ok(&spec.id) {
10415        return spec;
10416    }
10417    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10418    roster
10419        .get(from + 1..)
10420        .into_iter()
10421        .flatten()
10422        .chain(roster.iter())
10423        .find(|s| ok(&s.id))
10424        .cloned()
10425        .unwrap_or(spec)
10426}
10427
10428/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
10429/// seed so a CLI that mints its session id up front (`--session-id`) never
10430/// reuses the uuid the previous agent already opened under the same seat key.
10431pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10432    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10433}
10434
10435/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
10436/// so the classifier below and the code that builds the message cannot drift.
10437const TIMED_OUT: &str = "timed out";
10438
10439impl FailClass {
10440    /// `None` for an answer; otherwise how the turn failed.
10441    fn of(out: &AgentOutcome) -> Option<Self> {
10442        match out {
10443            AgentOutcome::Ok(_) => None,
10444            AgentOutcome::Quota(_) => Some(Self::Quota),
10445            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10446            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10447            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10448            // Not a failure of the agent. Callers check for it first; `None`
10449            // here means "no class to hand over on".
10450            AgentOutcome::ResourceBlocked(_) => None,
10451        }
10452    }
10453
10454    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
10455    fn stem_word(&self) -> &'static str {
10456        match self {
10457            Self::Quota => "quota",
10458            _ => "handover",
10459        }
10460    }
10461}
10462
10463/// The message's first line with its variable parts removed — digit runs and
10464/// path-like tokens — so "exit code 2" and "exit code 7" read
10465/// as one kind of failure.
10466fn failure_signature(msg: &str) -> String {
10467    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10468    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
10469    // the historical machine key while the process diagnosis uses readable prose.
10470    if let Some(code) = line
10471        .strip_prefix("process exited with no usable output (exit code ")
10472        .and_then(|s| s.strip_suffix(')'))
10473        .and_then(|s| s.parse::<i32>().ok())
10474    {
10475        line = format!("exited with some({code}) and no usable output");
10476    } else if matches!(
10477        line.as_str(),
10478        "process exited with no usable output (no exit code reported)"
10479            | "process exited with no usable output (exit code unavailable)"
10480    ) {
10481        line = "exited with none and no usable output".to_owned();
10482    }
10483    let mut out = Vec::new();
10484    for word in line.split_whitespace() {
10485        if word.contains('/') || word.contains('\\') {
10486            out.push("<path>".to_owned());
10487            continue;
10488        }
10489        let mut w = String::new();
10490        let mut in_digits = false;
10491        for c in word.chars() {
10492            if c.is_ascii_digit() {
10493                if !in_digits {
10494                    w.push('#');
10495                }
10496                in_digits = true;
10497            } else {
10498                in_digits = false;
10499                w.push(c);
10500            }
10501        }
10502        out.push(w);
10503    }
10504    out.join(" ").chars().take(120).collect()
10505}
10506
10507/// Whether a seat that just failed with `cur` may go to the next roster agent.
10508/// A quota or a timeout always may. Any other failure may not when the agent
10509/// before it failed the same way: an error the prompt causes would otherwise
10510/// walk the whole roster. `prev` is the class of the immediately preceding
10511/// agent's failure, so a quota or timeout in between breaks the run of
10512/// identical failures by itself.
10513fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10514    match cur {
10515        FailClass::Quota | FailClass::Timeout => true,
10516        FailClass::Other(_) => prev != Some(cur),
10517    }
10518}
10519
10520/// A short human reason for a failed outcome, for the handover record.
10521fn fail_reason(out: &AgentOutcome) -> String {
10522    match out {
10523        AgentOutcome::Ok(_) => String::new(),
10524        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10525        AgentOutcome::Dropped(o) => format!(
10526            "the CLI dropped the stream ({})",
10527            o.dropped
10528                .as_ref()
10529                .map(|d| d.why.as_str())
10530                .unwrap_or("it ended without delivering its answer")
10531        ),
10532        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10533            e.lines().next().unwrap_or("").chars().take(160).collect()
10534        }
10535    }
10536}
10537
10538/// Note one handover in the run: the structured record and, in the timeline,
10539/// the sentence a person reads. A quota keeps the wording it always had.
10540pub(crate) fn record_handover(
10541    state: &mut RunState,
10542    node: &str,
10543    seat: &str,
10544    from: &str,
10545    to: &str,
10546    class: &FailClass,
10547    reason: &str,
10548) {
10549    let message = if *class == FailClass::Quota {
10550        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10551    } else {
10552        format!("{seat}: handed over {from} -> {to} ({reason})")
10553    };
10554    state.event(node, message);
10555    state.handovers.push(Handover {
10556        at: Timestamp::now(),
10557        node: node.to_owned(),
10558        seat: seat.to_owned(),
10559        from: from.to_owned(),
10560        to: to.to_owned(),
10561        reason: reason.to_owned(),
10562    });
10563}
10564
10565/// Did this reply report running a command whose own CLI never confirmed an
10566/// exit status?
10567///
10568/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
10569/// command *finished* (see that type's own doc), so this can only be `true`
10570/// for a command whose completion event carried no readable exit code — not
10571/// for one that simply is not mentioned at all. That is the one signal this
10572/// crate can read, from the same record `state.jobs` renders, about a reply
10573/// standing next to work its own CLI cannot vouch for finishing; it is
10574/// deliberately not a check on the exit code's *value* (a fixer legitimately
10575/// runs a command that fails mid-iteration before it succeeds) and not a
10576/// guess at a command still running in the background (which emits no event
10577/// at all, and so leaves no evidence here to find).
10578fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10579    commands.iter().any(|c| c.exit_code.is_none())
10580}
10581
10582/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
10583/// trusted as a verified no-op — the adoption guard's own text-level half.
10584///
10585/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
10586/// timed out): a marker only earns the benefit of the doubt from a turn the
10587/// CLI itself vouches for finishing properly, the same house style
10588/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
10589/// *fix* report to for `commands`. A candidate that timed out, exited
10590/// non-zero, or left a command unconfirmed is read as the ordinary loss it
10591/// is, whatever prose it wrote — this returns `None` before it ever looks at
10592/// `text`. The remaining guards (the tree really is empty, the evidence is
10593/// non-empty) are the caller's: this only reads what the reply *claimed*.
10594fn verified_noop_claim(
10595    usable: bool,
10596    commands: &[agent::CommandEvidence],
10597    text: &str,
10598) -> Option<String> {
10599    (usable && !has_unconfirmed_command(commands))
10600        .then(|| verdict::verified_noop(text))
10601        .flatten()
10602}
10603
10604fn short(commit: &str) -> String {
10605    commit.chars().take(7).collect()
10606}
10607
10608fn make_executable(path: &Path) -> Result<()> {
10609    #[cfg(unix)]
10610    {
10611        use std::os::unix::fs::PermissionsExt as _;
10612        let mut perms = std::fs::metadata(path)?.permissions();
10613        perms.set_mode(0o755);
10614        std::fs::set_permissions(path, perms)?;
10615    }
10616    #[cfg(not(unix))]
10617    {
10618        let _ = path;
10619    }
10620    Ok(())
10621}
10622
10623/// What every seat in one batch shares: where the answers are attributed, the
10624/// prompt overlay they inherit, and the build cache they are told to use.
10625///
10626/// A struct rather than four more parameters: `wave` also needs the run's
10627/// state (to record who is answering right now) and the attempt number, and
10628/// eight positional arguments is both unreadable and a clippy error.
10629struct WaveCtx<'a> {
10630    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
10631    /// paid for it.
10632    run: &'a str,
10633    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
10634    node: &'a str,
10635    prompts: &'a Prompts,
10636    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
10637    cache: Option<&'a Path>,
10638    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
10639    /// `JobRecord::round`. `None` for every other node.
10640    round: Option<usize>,
10641    /// Carry each seat's failed-agent history across waves (the review loop
10642    /// only): start-of-round priority and handover choice read
10643    /// [`RunState::seat_history`], and every answer or failure writes it.
10644    carry_seats: bool,
10645    /// The run's park / interrupt requests, so a wait for a shared resource
10646    /// can be abandoned the moment the operator asks to park.
10647    pauses: Option<(&'a Pause, &'a Pause)>,
10648}
10649
10650/// Shared scheduling policy for operations and spawned seat invocations.
10651fn repository_wait(
10652    state: &RunState,
10653    pauses: Option<(&Pause, &Pause)>,
10654) -> crate::repo_lock::WaitPolicy {
10655    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10656    let inherited = crate::repo_lock::WaitPolicy::current();
10657    crate::repo_lock::WaitPolicy::new(
10658        Duration::from_secs(state.config.graph.timeout_resource_wait),
10659        move || {
10660            inherited.cancelled()
10661                || pauses
10662                    .as_ref()
10663                    .is_some_and(|(a, b)| a.parked() || b.parked())
10664        },
10665    )
10666}
10667
10668fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10669    if error.is::<crate::repo_lock::ResourceBlocked>() {
10670        state.resource_block = Some(crate::run::ResourceBlock {
10671            cause: crate::run::ResourceBlockCause::Other,
10672            node: "repository".into(),
10673            kind: if error.to_string().contains("disk pressure") {
10674                "disk pressure"
10675            } else if error.to_string().contains("disk admission") {
10676                "disk admission"
10677            } else if error.to_string().contains(crate::incremental::BLOCKED) {
10678                "incremental cache admission"
10679            } else {
10680                "repository coordination"
10681            }
10682            .into(),
10683            detail: error.to_string(),
10684            at: Timestamp::now(),
10685        });
10686        state.event("resource", error.to_string());
10687    }
10688}
10689
10690async fn coordinate_operation(
10691    state: &mut RunState,
10692    cwd: &Path,
10693    wait: &crate::repo_lock::WaitPolicy,
10694) -> Result<Option<crate::repo_lock::SeatLease>> {
10695    let result =
10696        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10697            .await;
10698    if let Err(e) = &result {
10699        record_repository_block(state, e);
10700    }
10701    result
10702}
10703
10704/// Run one job, honouring the parallelism budget.
10705async fn run_one(
10706    job: SeatJob,
10707    sem: Arc<Semaphore>,
10708    ctx: &WaveCtx<'_>,
10709    state: &mut RunState,
10710    attempt: usize,
10711) -> (SeatState, AgentOutcome) {
10712    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10713        .await
10714        .pop()
10715        .expect("one job in, one result out");
10716    (seat, out)
10717}
10718
10719/// Run every job concurrently, capped by the semaphore, preserving order.
10720///
10721/// Every seat in the batch is recorded into [`RunState::active`] before the
10722/// wave starts and cleared as each answer lands, so the run's own record says
10723/// who is still being waited on rather than only who finished.
10724async fn wave(
10725    jobs: Vec<SeatJob>,
10726    sem: Arc<Semaphore>,
10727    ctx: &WaveCtx<'_>,
10728    state: &mut RunState,
10729    attempt: usize,
10730) -> Vec<(usize, SeatState, AgentOutcome)> {
10731    let WaveCtx {
10732        run,
10733        node,
10734        prompts,
10735        cache,
10736        round,
10737        carry_seats: _,
10738        pauses,
10739    } = *ctx;
10740    for job in &jobs {
10741        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10742    }
10743    if let Err(e) = state.save() {
10744        // A failed persist of "who is answering right now" must not abort the
10745        // wave: the seats are already being asked, and the alternative is
10746        // losing the answers to save a status line nobody may even be
10747        // watching.
10748        tracing::warn!("could not persist in-progress seats: {e:#}");
10749    }
10750    // Hold the shared build cache's lease for the whole batch, not per job:
10751    // several candidates (an implement wave) or a fixer legitimately share
10752    // one cache concurrently within this run, and that stays untouched — a
10753    // single lease taken once for the whole wave and released once it is
10754    // done is what stops a *different* borrower (another run's own wave, its
10755    // e2e/gate, a human's `magi review`) from interleaving a build into the
10756    // same directory while this one is in flight. Best-effort, not
10757    // all-or-nothing: a wave that cannot get the lease within its own
10758    // longest job's budget still runs — an hour of paid implementer calls is
10759    // not thrown away over cache contention — but every write-allowed seat
10760    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
10761    // below), the same fallback a read-only seat always gets, rather than
10762    // building into a directory this run was never granted. The identity
10763    // record is still invalidated below either way, so the next tracked
10764    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
10765    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10766    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10767    // never borrowed from a job's agent budget (and never charged to it either,
10768    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10769    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10770    let cache_guard = if let Some(cache_dir) = cache {
10771        if jobs_had_a_writer {
10772            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10773            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10774            match acquire_cache_lease(
10775                state,
10776                cache_dir,
10777                &owner,
10778                resource_limit,
10779                node,
10780                node,
10781                &cancel,
10782            )
10783            .await
10784            {
10785                Ok(Some(g)) => Some(g),
10786                // Parking: leave without spawning anything. The run parks at
10787                // the next boundary and the seats are asked afresh on resume.
10788                Ok(None) => {
10789                    if let Some(block) = &mut state.resource_block {
10790                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10791                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10792                        };
10793                    }
10794                    return block_wave(
10795                        jobs,
10796                        state,
10797                        node,
10798                        attempt,
10799                        "parked while waiting for the shared build cache",
10800                    );
10801                }
10802                // An unreadable lease is not contention: keep the old
10803                // fallback (no shared cache for this wave's seats).
10804                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10805                Err(e) => {
10806                    // Writing seats may not build into a cache this run does
10807                    // not hold, and an agent that cannot be given one is not
10808                    // asked at all: the run stops resumably instead.
10809                    let why = format!("{e:#}");
10810                    state.event(
10811                        "resource",
10812                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10813                    );
10814                    return block_wave(jobs, state, node, attempt, &why);
10815                }
10816            }
10817        } else {
10818            None
10819        }
10820    } else {
10821        None
10822    };
10823    // Every job keeps its whole budget: the wait above was not the agent's.
10824    // Restart the seats' clocks so "started" means "being spawned now".
10825    state.seats_spawning(node, attempt);
10826    if let Err(e) = state.save() {
10827        tracing::warn!("could not persist the start of a wave: {e:#}");
10828    }
10829    // Queueing behind this wave's own seats for a parallelism slot is normal
10830    // (each of them is bounded by its own agent timeout); only a wait beyond
10831    // all of them plus the resource limit means the slots never came back.
10832    let slot_limit = resource_limit
10833        + jobs
10834            .iter()
10835            .map(|j| j.timeout)
10836            .max()
10837            .unwrap_or_default()
10838            .saturating_mul(jobs.len() as u32);
10839    let mut set = tokio::task::JoinSet::new();
10840    // Seats announce themselves here the moment they hold a parallelism
10841    // permit, i.e. just before their CLI is spawned.
10842    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10843    let overlay = prompts.overlay(node);
10844    let repository_wait = repository_wait(state, pauses);
10845    for (i, mut job) in jobs.into_iter().enumerate() {
10846        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10847        if cache.is_some() {
10848            job.prompt.push('\n');
10849            job.prompt
10850                .push_str(&prompt::build_cache_note(node, job.allow_write));
10851        }
10852        let sem = Arc::clone(&sem);
10853        let run = run.to_owned();
10854        let node = node.to_owned();
10855        // Only implementers were told about the task's attachments, so only
10856        // their seats get the directory widened for reading.
10857        let attachments = if node == "implement" {
10858            state.attachments.clone()
10859        } else {
10860            Vec::new()
10861        };
10862        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10863        // `prompt::build_cache_note`'s doc for why setting it anyway is
10864        // exactly how a sandboxed reviewer's write refusal got reported as a
10865        // defect in the patch, not a property of its own seat. And a
10866        // write-allowed one is handed it only when the lease above was
10867        // actually acquired: a wave that could not get it (`cache_guard` is
10868        // `None`, see its own comment) must not send seats to build into a
10869        // directory this run does not hold - that is the exact concurrent,
10870        // unmanaged-write race this module exists to prevent, not something
10871        // "proceeding anyway" is allowed to reintroduce.
10872        let cache = cache
10873            .filter(|_| job.allow_write && cache_guard.is_some())
10874            .map(Path::to_path_buf);
10875        let spawned_tx = spawned_tx.clone();
10876        let repository_wait = repository_wait.clone();
10877        let coordination = state.config.coordination.clone();
10878        let fetched_base = state.fetched_base.clone();
10879        set.spawn(async move {
10880            // A wait for a parallelism slot is scheduling, not the agent: it
10881            // is bounded by its own limit and ends as a resource block, never
10882            // as a timeout of the agent that was not started.
10883            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10884                Ok(p) => p,
10885                Err(_) => {
10886                    return (
10887                        i,
10888                        job.seat,
10889                        AgentOutcome::ResourceBlocked(format!(
10890                            "no parallel slot became free within {}s",
10891                            slot_limit.as_secs()
10892                        )),
10893                        false,
10894                    );
10895                }
10896            };
10897            spawned_tx.send(job.seat.key.clone()).ok();
10898            let mut seat = job.seat;
10899            let out = repository_wait
10900                .scope(agent::invoke(
10901                    &job.spec,
10902                    &mut seat,
10903                    &Invocation {
10904                        coordination: coordination.as_ref(),
10905                        fetched_base: fetched_base.as_ref(),
10906                        cwd: &job.cwd,
10907                        prompt: &job.prompt,
10908                        timeout: job.timeout,
10909                        allow_write: job.allow_write,
10910                        sessions: job.sessions,
10911                        artifacts: &job.artifacts,
10912                        stem: &job.stem,
10913                        run: &run,
10914                        node: &node,
10915                        cache_dir: cache.as_deref(),
10916                        attachments: &attachments,
10917                        writable: &[],
10918                    },
10919                ))
10920                .await;
10921            let interrupted = out
10922                .as_ref()
10923                .err()
10924                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10925            let out = match out {
10926                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10927                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10928                // Billed work the CLI failed to hand over is not an ordinary
10929                // failure, but its text is the CLI's raw error JSON, not an
10930                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
10931                // read it as one by forgetting to check. `usable()` is always
10932                // false here (dropped implies an empty response), so this has
10933                // to be checked before the catch-all `Failed` below or the
10934                // one shape this exists for is lost with the rest.
10935                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10936                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10937                Ok(o) => AgentOutcome::Failed(format!(
10938                    "process exited with no usable output ({})",
10939                    crate::notices::exit_status(o.exit_code)
10940                )),
10941                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10942                    AgentOutcome::ResourceBlocked(e.to_string())
10943                }
10944                Err(e) => AgentOutcome::Failed(e.to_string()),
10945            };
10946            (i, seat, out, interrupted)
10947        });
10948    }
10949    drop(spawned_tx);
10950    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10951    loop {
10952        let joined = tokio::select! {
10953            Some(key) = spawned_rx.recv() => {
10954                // The clock of this seat starts now, not when it queued.
10955                state.seat_spawned(&key);
10956                if let Err(e) = state.save() {
10957                    tracing::warn!("could not persist a seat's start: {e:#}");
10958                }
10959                continue;
10960            }
10961            joined = set.join_next() => joined,
10962        };
10963        let Some(joined) = joined else {
10964            break;
10965        };
10966        let (i, seat, out, interrupted) = match joined {
10967            Ok(v) => v,
10968            // No seat to clear: a panicked task never reported which one it
10969            // was. The defensive sweep below this loop is what stops that
10970            // seat's `active` entry from surviving forever.
10971            Err(e) => {
10972                tracing::error!("agent task panicked: {e}");
10973                continue;
10974            }
10975        };
10976        state.seat_finished(&seat.key);
10977        if interrupted
10978            && node == "implement"
10979            && let Some(label) = state
10980                .candidates
10981                .iter()
10982                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10983                .map(|candidate| candidate.label)
10984        {
10985            state.candidate_preservation.insert(
10986                label,
10987                "interrupted implementation awaiting persistence".into(),
10988            );
10989            state.interrupted_implementations.insert(label);
10990        }
10991        if let AgentOutcome::ResourceBlocked(why) = &out
10992            && state.resource_block.is_none()
10993        {
10994            state.resource_block = Some(crate::run::ResourceBlock {
10995                cause: crate::run::ResourceBlockCause::Other,
10996                node: node.to_owned(),
10997                kind: if why.contains("disk pressure") {
10998                    "disk pressure"
10999                } else if why.contains("disk admission") {
11000                    "disk admission"
11001                } else if why.contains(crate::incremental::BLOCKED) {
11002                    "incremental cache admission"
11003                } else if why.contains("repository coordination") {
11004                    "repository coordination"
11005                } else {
11006                    "parallel slot"
11007                }
11008                .to_owned(),
11009                detail: why.clone(),
11010                at: Timestamp::now(),
11011            });
11012            let activity = if interrupted {
11013                "agent execution interrupted"
11014            } else {
11015                "no agent was started"
11016            };
11017            state.event("resource", format!("{node}: {activity} - {why}"));
11018        }
11019        record_jobs(state, node, round, &seat.key, &out);
11020        if let Err(e) = state.save() {
11021            tracing::warn!("could not persist a seat's completion: {e:#}");
11022        }
11023        if collected.len() <= i {
11024            collected.resize_with(i + 1, || None);
11025        }
11026        collected[i] = Some((i, seat, out));
11027    }
11028    // Belt-and-braces for the panic branch above: every seat this exact batch
11029    // started shares this `(node, attempt)` pair, and every seat that finished
11030    // normally already cleared itself, so anything left tagged with it here
11031    // can only be a panicked task's leftover. Cleared unconditionally rather
11032    // than left to read as still answering forever.
11033    if state
11034        .active
11035        .values()
11036        .any(|a| a.node == node && a.attempt == attempt)
11037    {
11038        state
11039            .active
11040            .retain(|_, a| !(a.node == node && a.attempt == attempt));
11041        if let Err(e) = state.save() {
11042            tracing::warn!("could not persist the end of a wave: {e:#}");
11043        }
11044    }
11045    // Whether or not the lease above was actually held, several worktrees
11046    // may just have built into the cache with nothing here able to name one
11047    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
11048    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
11049    // clean it might not have strictly needed; trusting a stale match would
11050    // cost it a wrong answer.
11051    if let Some(cache_dir) = cache
11052        && jobs_had_a_writer
11053    {
11054        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11055    }
11056    if let Some(guard) = cache_guard {
11057        guard.release();
11058    }
11059    collected.into_iter().flatten().collect()
11060}
11061
11062/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
11063/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
11064/// this is data collection, not the fix-specific completion contract in
11065/// [`Runner::continue_fix_report`], and applies regardless of which node
11066/// asked.
11067///
11068/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
11069/// evidence from; `Failed` does not, and correctly contributes nothing — a
11070/// timeout or crash is not itself evidence about a command the seat may have
11071/// started.
11072fn record_jobs(
11073    state: &mut RunState,
11074    node: &str,
11075    round: Option<usize>,
11076    seat: &str,
11077    out: &AgentOutcome,
11078) {
11079    let commands: &[agent::CommandEvidence] = match out {
11080        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11081        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11082    };
11083    let checked_at = Timestamp::now();
11084    for c in commands {
11085        state.jobs.push(JobRecord {
11086            node: node.to_owned(),
11087            round,
11088            seat: seat.to_owned(),
11089            id: c.id.clone(),
11090            description: c.description.clone(),
11091            checked_at,
11092            status: match c.exit_code {
11093                Some(0) => JobStatus::Completed,
11094                Some(_) => JobStatus::Failed,
11095                None => JobStatus::Unknown,
11096            },
11097            exit_code: c.exit_code,
11098            result_summary: c.result_summary.clone(),
11099            source: c.source.clone(),
11100        });
11101    }
11102}
11103
11104/// Is a review round clean, given how many reviewer seats answered against
11105/// how many the round expected?
11106///
11107/// A seat that never answered (timeout, crash, unparsable output) is not a
11108/// seat that read the patch and found nothing — treating it as such is
11109/// exactly the bug this function exists to close. Under the default `block`
11110/// policy a missing seat can never be clean; `warn` still requires the seats
11111/// that *did* answer to have found nothing blocking and verification to be
11112/// green.
11113///
11114/// `quota_missing` narrows that `block` default for exactly one cause of
11115/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
11116/// a session limit lifts by the very next round buys nothing — the seat is
11117/// asked again with the same quota — so once every missing seat is accounted
11118/// for by a quota loss (and at least one seat *did* answer, so a decision has
11119/// something to rest on) the round is decided on the panel that could answer,
11120/// same as `warn` would. A panel that lost every seat to quota is not
11121/// decided here: `answered == 0` falls through to the existing `block`
11122/// fallback so a fully collapsed panel still waits rather than landing on no
11123/// review at all.
11124fn round_is_clean(
11125    blocking: usize,
11126    e2e_ok: bool,
11127    answered: usize,
11128    expected: usize,
11129    quota_missing: usize,
11130    policy: IncompleteReviewPolicy,
11131) -> bool {
11132    if blocking != 0 || !e2e_ok {
11133        return false;
11134    }
11135    if answered == expected || policy == IncompleteReviewPolicy::Warn {
11136        return true;
11137    }
11138    answered > 0 && expected - answered <= quota_missing
11139}
11140
11141/// The review loop's own conclusion, derived entirely from its persisted
11142/// round records and the round budget that produced them — never from
11143/// `status`, so a reentry (or `gate`/`merge` reading it independently)
11144/// recomputes the identical answer regardless of what an earlier node in the
11145/// same walk, or a previous walk, did to `status`.
11146///
11147/// `None` while more rounds remain to try, including when review never ran
11148/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
11149/// gone clean, or the budget is spent, or the tree has stopped moving (see
11150/// [`STAGNANT_LIMIT`]), the answer is one of two things:
11151///
11152/// - An incomplete panel that raised nothing is missing input, not a
11153///   verified tree — never a hand-off candidate, whatever verification said
11154///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
11155/// - Otherwise, green e2e on the last round hands off (see
11156///   [`Runner::stop_reviewing`]); red e2e blocks.
11157///
11158/// A last round whose own verification is still `ResourceBlocked` — magi
11159/// itself never got a command to run, not evidence the patch is broken —
11160/// is neither: this returns `None` for it too, the same as "more rounds
11161/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
11162/// handling of that shape) instead of this cheap recomputation guessing a
11163/// verdict a real attempt never produced.
11164pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11165    if max_rounds == 0
11166        || reviews.last().is_some_and(|r| {
11167            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11168        })
11169    {
11170        return Some(RunStatus::Gating);
11171    }
11172    let last = reviews.last()?;
11173    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11174    if reviews.len() < max_rounds && !stagnant {
11175        return None;
11176    }
11177    if last.incomplete() && last.blocking == 0 {
11178        return Some(RunStatus::Blocked);
11179    }
11180    if last.e2e_status() == E2eStatus::ResourceBlocked
11181        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11182    {
11183        return None;
11184    }
11185    Some(if last.e2e.iter().all(CommandOutcome::ok) {
11186        RunStatus::Gating
11187    } else {
11188        RunStatus::Blocked
11189    })
11190}
11191
11192/// How long a re-ask may take, given the budget the first attempt had.
11193///
11194/// A `nudged` retry is a request to restate an answer the seat has already
11195/// worked out: it carries no new work, so it does not deserve the original
11196/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
11197/// 133 seconds while a third sat for over ten minutes on a resumed session
11198/// holding 410 KB of prior output - and because the retry had inherited the
11199/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
11200/// judging round whose other seats were long finished.
11201///
11202/// A quarter of the budget, with a floor so that a deliberately short timeout
11203/// does not collapse to nothing. A retry that re-sends the whole prompt
11204/// (because the seat kept no context) is the original job again, and keeps the
11205/// original budget.
11206pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11207    if nudged {
11208        (full / 4).max(Duration::from_secs(120)).min(full)
11209    } else {
11210        full
11211    }
11212}
11213
11214/// Run a wave and parse each reply, re-asking the seats whose reply was
11215/// unusable.
11216///
11217/// The re-ask is a nudge rather than the whole prompt again when the seat still
11218/// holds its conversation, which is the difference between a cheap retry and
11219/// paying for the entire candidate set twice.
11220///
11221/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
11222/// successor in `roster` is handed to it instead of being re-asked: the
11223/// handover is the retry. A seat with no successor left (a single-agent
11224/// roster, the roster's tail) is nudged as before, up to `retries` times. So
11225/// the asks to one seat in one node number at most
11226/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
11227/// is asked once (a dropped stream is nudged first), and only the last agent
11228/// of the chain gets the `retries` same-agent nudges. Each roster agent is
11229/// tried at most once per seat, walking forward from the seat's own position and never wrapping
11230/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
11231/// other failure stops the chain when the previous agent failed the same way
11232/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
11233/// [`has_context`] is false and the job's own full prompt and full budget are
11234/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
11235/// intermediate ones are not losses) and is returned as a failure like any
11236/// other absent seat — the caller decides whether the panel still has a
11237/// quorum. An empty `roster` disables handover: failures are nudged as they
11238/// always were, and a quota is simply lost. A reply that fails to parse or
11239/// validate is the prompt's doing and is only ever nudged, never handed over.
11240///
11241/// The returned [`SeatState`] names the agent that answered (or tried last).
11242#[allow(clippy::too_many_arguments)]
11243async fn ask_json_wave<T>(
11244    jobs: Vec<SeatJob>,
11245    sem: Arc<Semaphore>,
11246    retries: usize,
11247    roster: &[AgentSpec],
11248    ctx: &WaveCtx<'_>,
11249    losses: &mut Vec<QuotaLoss>,
11250    state: &mut RunState,
11251    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11252) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11253where
11254    T: serde::de::DeserializeOwned + Send + 'static,
11255{
11256    ask_wave_with(
11257        jobs,
11258        sem,
11259        retries,
11260        roster,
11261        ctx,
11262        losses,
11263        state,
11264        &|text: &str| {
11265            let v = verdict::extract_json::<T>(text)?;
11266            validate(&v)?;
11267            Ok(v)
11268        },
11269    )
11270    .await
11271}
11272
11273/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
11274/// a node whose answer is prose (deliberation) shares the same handover,
11275/// failure classification, quota bookkeeping and bounds instead of a copy.
11276///
11277/// A seat handed to another roster agent is sent the job's `handover` prompt
11278/// (when it has one) rather than `prompt`: the new agent has no session, so a
11279/// resume-style prompt would be incomplete. That holds for the handover ask
11280/// and for every nudge to that agent whose `has_context` is false.
11281#[allow(clippy::too_many_arguments)]
11282async fn ask_wave_with<T>(
11283    jobs: Vec<SeatJob>,
11284    sem: Arc<Semaphore>,
11285    retries: usize,
11286    roster: &[AgentSpec],
11287    ctx: &WaveCtx<'_>,
11288    losses: &mut Vec<QuotaLoss>,
11289    state: &mut RunState,
11290    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11291) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11292where
11293    T: Send + 'static,
11294{
11295    let n = jobs.len();
11296    let originals: Vec<SeatJob> = jobs;
11297    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11298    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11299    // Nudges each seat's *current* agent has taken — 0 for a first-ask
11300    // answer, N once it has gone through N nudges. Read back once this
11301    // returns, so a caller building a history record (`ReviewRecord`) can
11302    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
11303    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
11304    // field's own doc.
11305    let mut nudges: Vec<usize> = vec![0; n];
11306    // The agent now occupying each seat, the ids it has already been through,
11307    // where in the roster the walk began, the class of the last failure, and
11308    // the stem word of a handover not yet asked (full prompt, full budget).
11309    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11310    let mut tried: Vec<BTreeSet<String>> = specs
11311        .iter()
11312        .map(|s| BTreeSet::from([s.id.clone()]))
11313        .collect();
11314    let starts: Vec<usize> = specs
11315        .iter()
11316        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11317        .collect();
11318    let carry = ctx.carry_seats && !roster.is_empty();
11319    // Carried across rounds: ids that failed the seat earlier, and how the
11320    // last failure went (so a repeat of it is not handed over again).
11321    let carried: Vec<BTreeSet<String>> = originals
11322        .iter()
11323        .map(|j| {
11324            state
11325                .seat_history
11326                .get(&j.seat.key)
11327                .filter(|_| carry)
11328                .map(|h| h.failed.clone())
11329                .unwrap_or_default()
11330        })
11331        .collect();
11332    let mut prev: Vec<Option<FailClass>> = originals
11333        .iter()
11334        .map(|j| {
11335            state
11336                .seat_history
11337                .get(&j.seat.key)
11338                .filter(|_| carry)
11339                .and_then(|h| h.last_fail.clone())
11340        })
11341        .collect();
11342    let next_agent =
11343        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11344            let others: BTreeSet<String> = specs
11345                .iter()
11346                .enumerate()
11347                .filter(|(j, _)| *j != i)
11348                .map(|(_, s)| s.id.clone())
11349                .collect();
11350            pick_successor(
11351                roster,
11352                starts[i],
11353                tried,
11354                carry.then(|| &carried[i]),
11355                &others,
11356            )
11357            .cloned()
11358        };
11359    let mut fresh: Vec<Option<String>> = vec![None; n];
11360    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11361    let mut pending: Vec<usize> = (0..n).collect();
11362
11363    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
11364    // asks: an agent with a successor is asked once and handed over, and only
11365    // a seat with no successor spends `retries` nudges on the same agent. This
11366    // only guarantees the loop's own termination whatever those say.
11367    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11368    for round in 0..max_rounds {
11369        if pending.is_empty() {
11370            break;
11371        }
11372        let mut batch = Vec::with_capacity(pending.len());
11373        let mut renudged: Vec<&str> = Vec::new();
11374        for &i in &pending {
11375            let src = &originals[i];
11376            // A seat now held by another agent than the job named has no
11377            // session of its own: it gets the full-context prompt whenever
11378            // it is asked in full (the handover ask, a nudge it cannot
11379            // resume).
11380            let full: &str = match &src.handover {
11381                Some(h) if specs[i].id != src.spec.id => h,
11382                _ => &src.prompt,
11383            };
11384            // The prompt and the budget are one decision: a nudge restates
11385            // finished work, a re-sent prompt redoes it.
11386            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11387                (
11388                    full.to_owned(),
11389                    src.timeout,
11390                    format!("{}-{word}-{}", src.stem, specs[i].id),
11391                )
11392            } else if nudges[i] == 0 {
11393                (full.to_owned(), src.timeout, src.stem.clone())
11394            } else {
11395                renudged.push(src.seat.key.as_str());
11396                let why = done[i]
11397                    .as_ref()
11398                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
11399                    .unwrap_or_else(|| "no parsable answer".to_owned());
11400                let nudge = prompt::nudge(&why);
11401                let nudged = has_context(&specs[i], &seats[i], src.sessions);
11402                let prompt = if nudged {
11403                    nudge
11404                } else {
11405                    format!("{full}\n\n---\n\n{nudge}")
11406                };
11407                (
11408                    prompt,
11409                    retry_budget(src.timeout, nudged),
11410                    format!("{}-retry{}", src.stem, nudges[i]),
11411                )
11412            };
11413            batch.push(SeatJob {
11414                spec: specs[i].clone(),
11415                seat: seats[i].clone(),
11416                cwd: src.cwd.clone(),
11417                prompt,
11418                timeout,
11419                allow_write: src.allow_write,
11420                sessions: src.sessions,
11421                artifacts: src.artifacts.clone(),
11422                stem,
11423                handover: None,
11424            });
11425        }
11426
11427        if !renudged.is_empty() {
11428            state.event(
11429                ctx.node,
11430                format!("retry {round}: re-asking {}", renudged.join(", ")),
11431            );
11432        }
11433        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11434        let mut still = Vec::new();
11435        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11436            seats[i] = seat;
11437            // No agent was asked: nothing to hand over, nudge, remember or
11438            // charge. The caller sees `resource_block` and stops the node.
11439            if let AgentOutcome::ResourceBlocked(why) = &out {
11440                done[i] = Some(Err(anyhow::anyhow!("{why}")));
11441                continue;
11442            }
11443            let class = FailClass::of(&out);
11444            // A dropped stream is nudged first (the conversation is still
11445            // there to pick up); only a seat whose nudges are spent hands over.
11446            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11447                && nudges[i] < retries
11448                && !roster.is_empty();
11449            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11450                let next = should_hand_over(prev[i].as_ref(), &cur)
11451                    .then(|| next_agent(i, &tried[i], &specs))
11452                    .flatten();
11453                if carry {
11454                    let h = state
11455                        .seat_history
11456                        .entry(originals[i].seat.key.clone())
11457                        .or_default();
11458                    h.failed.insert(specs[i].id.clone());
11459                    h.last_fail = Some(cur.clone());
11460                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11461                        h.last_ok = None;
11462                    }
11463                    // Saved before the next agent is asked, so a restart in
11464                    // between does not forget who failed.
11465                    if let Err(e) = state.save() {
11466                        tracing::warn!("could not persist a seat's failure history: {e:#}");
11467                    }
11468                }
11469                if let Some(next) = next {
11470                    record_handover(
11471                        state,
11472                        ctx.node,
11473                        &originals[i].seat.key,
11474                        &specs[i].id,
11475                        &next.id,
11476                        &cur,
11477                        &fail_reason(&out),
11478                    );
11479                    tried[i].insert(next.id.clone());
11480                    prev[i] = Some(cur.clone());
11481                    seats[i] =
11482                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11483                    specs[i] = next;
11484                    fresh[i] = Some(cur.stem_word().to_owned());
11485                    nudges[i] = 0;
11486                    done[i] = Some(Err(anyhow::anyhow!(
11487                        "handed over after: {}",
11488                        fail_reason(&out)
11489                    )));
11490                    still.push(i);
11491                    continue;
11492                }
11493            }
11494            if carry
11495                && !nudge_first
11496                && let Some(cur) = class.clone()
11497            {
11498                // The chain ended here (no successor, or a repeated failure).
11499                let h = state
11500                    .seat_history
11501                    .entry(originals[i].seat.key.clone())
11502                    .or_default();
11503                h.failed.insert(specs[i].id.clone());
11504                h.last_fail = Some(cur);
11505            }
11506            let parsed = match out {
11507                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11508                AgentOutcome::Quota(o) => {
11509                    last_quota[i] = o.quota.clone();
11510                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11511                }
11512                // Not a parseable answer: the nudge loop re-asks it, which is
11513                // exactly what a dropped stream needs. Just don't hand its raw
11514                // error JSON to `extract_json`.
11515                AgentOutcome::Dropped(o) => {
11516                    let why = o
11517                        .dropped
11518                        .as_ref()
11519                        .map(|d| d.why.as_str())
11520                        .unwrap_or("the CLI ended the stream without delivering its answer");
11521                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11522                }
11523                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11524                    Err(anyhow::anyhow!(e))
11525                }
11526            };
11527            let quota = class == Some(FailClass::Quota);
11528            let failed = parsed.is_err();
11529            if !failed && matches!(ctx.node, "judge" | "vote") {
11530                // A validated answer restores this seat's quorum presence,
11531                // including on the normal path after an all-seat block.
11532                state
11533                    .quota
11534                    .retain(|loss| loss.seat != originals[i].seat.key);
11535            }
11536            done[i] = Some(parsed);
11537            if carry && !failed {
11538                let h = state
11539                    .seat_history
11540                    .entry(originals[i].seat.key.clone())
11541                    .or_default();
11542                h.failed.remove(&specs[i].id);
11543                h.last_ok = Some(specs[i].id.clone());
11544                h.last_fail = None;
11545            }
11546            // Do not re-ask a rate-limited seat (quota) — a retry is known to
11547            // fail the same way; and never re-ask a seat that already parsed.
11548            // A failed agent that still has a successor is not re-asked
11549            // either: the handover was its remedy and has just been refused
11550            // (the chain stops on a repeated failure class). A seat with no
11551            // successor left (a single-agent roster, the roster's tail, or an
11552            // empty roster) keeps the same-agent nudge, bounded by `retries`.
11553            let agent_failure = class.is_some()
11554                && !nudge_first
11555                && !roster.is_empty()
11556                && next_agent(i, &tried[i], &specs).is_some();
11557            if failed && !quota && !agent_failure && nudges[i] < retries {
11558                nudges[i] += 1;
11559                still.push(i);
11560            }
11561        }
11562        pending = still;
11563    }
11564
11565    // One loss per seat whose chain ended on a quota: the intermediate ones
11566    // were absorbed by a handover and are not losses.
11567    for (i, q) in last_quota.into_iter().enumerate() {
11568        if let Some(quota) = q {
11569            losses.push(QuotaLoss::from_report(
11570                originals[i].seat.key.clone(),
11571                ctx.node,
11572                Some(&quota),
11573            ));
11574        }
11575    }
11576
11577    // A stalled quorum already has its own recovery checkpoint. Refresh
11578    // its missing seats before deciding whether it can resume.
11579    if state.status != RunStatus::Stalled
11580        && ctx.node != "deliberate"
11581        && n > 0
11582        && losses.len() == n
11583        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11584    {
11585        block_machine_outage(state, ctx.node, losses);
11586        state.quota.append(losses);
11587    }
11588
11589    seats
11590        .into_iter()
11591        .zip(done)
11592        .zip(nudges)
11593        .map(|((seat, res), attempts)| {
11594            (
11595                seat,
11596                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11597                attempts,
11598            )
11599        })
11600        .collect()
11601}
11602
11603/// A node with no available agent has no task verdict. Preserve the node
11604/// checkpoint and record why a subsequent explicit resume is needed.
11605pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11606    let mut auth: Vec<_> = losses
11607        .iter()
11608        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11609        .map(|loss| loss.agent.clone())
11610        .collect();
11611    auth.sort();
11612    auth.dedup();
11613    let (kind, detail) = if auth.is_empty() {
11614        (
11615            "agent usage limit",
11616            "All seats reached their agent usage limits; resume after usage is available"
11617                .to_owned(),
11618        )
11619    } else {
11620        (
11621            "agent authentication",
11622            format!(
11623                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11624                auth.join(", ")
11625            ),
11626        )
11627    };
11628    state.resource_block = Some(crate::run::ResourceBlock {
11629        node: node.to_owned(),
11630        kind: kind.to_owned(),
11631        cause: crate::run::ResourceBlockCause::Other,
11632        detail: detail.clone(),
11633        at: Timestamp::now(),
11634    });
11635    state.event(node, detail);
11636}
11637
11638/// Did the wave end without asking any agent (see [`block_wave`])?
11639fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11640    results
11641        .iter()
11642        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11643}
11644
11645/// Settle a wave that never spawned anything: every job answers
11646/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
11647/// else about the seat (session, history, quota) is touched.
11648fn block_wave(
11649    jobs: Vec<SeatJob>,
11650    state: &mut RunState,
11651    node: &str,
11652    attempt: usize,
11653    why: &str,
11654) -> Vec<(usize, SeatState, AgentOutcome)> {
11655    state.resource_wait = None;
11656    state
11657        .active
11658        .retain(|_, a| !(a.node == node && a.attempt == attempt));
11659    if let Err(e) = state.save() {
11660        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11661    }
11662    jobs.into_iter()
11663        .enumerate()
11664        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11665        .collect()
11666}
11667
11668/// Acquire the shared build cache's lease, waiting out contention within
11669/// `budget` (never past it — see AGENTS.md's build-cache section on why an
11670/// unbounded wait is never acceptable).
11671///
11672/// A first, non-blocking check happens before ever waiting; if it finds the
11673/// lease busy, that fact is logged as a `verify` event *and* flushed with
11674/// [`RunState::save`] immediately — not only once the wait finally succeeds
11675/// or gives up — so a `magi show` run by a different process while this one
11676/// is still waiting reads a `run.json` that says so, rather than whatever it
11677/// looked like before the wait started. The same applies to the terminal
11678/// failure: logged and saved before this returns `Err`, so a caller that
11679/// could not get the lease at all still leaves a legible record of why.
11680async fn acquire_cache_lease(
11681    state: &mut RunState,
11682    cache_dir: &Path,
11683    owner: &crate::cache::Owner,
11684    budget: Duration,
11685    node: &str,
11686    context: &str,
11687    cancel: &(dyn Fn() -> bool + Sync),
11688) -> Result<Option<crate::cache::Guard>> {
11689    let home = crate::run::home();
11690    let started = Instant::now();
11691    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11692        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11693        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11694        Err(e) => {
11695            state.event(
11696                "verify",
11697                format!("{context}: could not check the shared build cache: {e:#}"),
11698            );
11699            if let Err(e2) = state.save() {
11700                tracing::warn!("could not persist a cache-check failure: {e2:#}");
11701            }
11702            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11703        }
11704    };
11705    state.event(
11706        "verify",
11707        format!(
11708            "{context}: waiting for the shared build cache at {} ({})",
11709            cache_dir.display(),
11710            busy.describe()
11711        ),
11712    );
11713    // Seats of this node are waiting, not running: say so, so the countdown a
11714    // reader sees only ever measures an agent that has been spawned.
11715    state.resource_wait = Some(crate::run::ResourceWait {
11716        node: context.to_owned(),
11717        kind: "shared build cache".to_owned(),
11718        holder: busy.describe(),
11719        since: Timestamp::now(),
11720        limit_secs: budget.as_secs(),
11721    });
11722    if let Err(e) = state.save() {
11723        tracing::warn!("could not persist a cache wait: {e:#}");
11724    }
11725    let remaining = budget.saturating_sub(started.elapsed());
11726    let waited = crate::cache::wait_for_unless(
11727        &home,
11728        cache_dir,
11729        owner,
11730        remaining,
11731        Duration::from_secs(5),
11732        cancel,
11733    )
11734    .await;
11735    state.resource_wait = None;
11736    let cause = match &waited {
11737        Ok(Some(_)) => None,
11738        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11739        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11740            Some(crate::run::ResourceBlockCause::Timeout {
11741                waited_secs: started.elapsed().as_secs(),
11742                limit_secs: budget.as_secs(),
11743            })
11744        }
11745        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11746    };
11747    if let Some(cause) = cause {
11748        let detail = match &waited {
11749            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11750            Err(error) => format!("{error:#}"),
11751        };
11752        state.resource_block = Some(crate::run::ResourceBlock {
11753            cause,
11754            node: node.to_owned(),
11755            kind: "shared build cache".into(),
11756            detail,
11757            at: Timestamp::now(),
11758        });
11759        state.event(
11760            "resource",
11761            state.resource_block.as_ref().unwrap().description(),
11762        );
11763    }
11764    if let Err(error) = state.save() {
11765        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11766    }
11767    waited
11768}
11769
11770/// The lease state could not be read at all (unreadable lease file, a
11771/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11772/// known to be contended, so a wave degrades to running without a shared cache
11773/// instead of stopping on a hold that [`crate::triage`] could never release.
11774#[derive(Debug)]
11775struct CacheCheckError(String);
11776
11777impl std::fmt::Display for CacheCheckError {
11778    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11779        f.write_str(&self.0)
11780    }
11781}
11782
11783impl std::error::Error for CacheCheckError {}
11784
11785/// Keep a specific acquisition outcome when the caller adds command evidence.
11786fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11787    state
11788        .resource_block
11789        .get_or_insert_with(|| crate::run::ResourceBlock {
11790            cause: crate::run::ResourceBlockCause::Other,
11791            node: node.to_owned(),
11792            kind: "shared build cache".into(),
11793            detail: format!("{error:#}"),
11794            at: Timestamp::now(),
11795        });
11796    if let Err(error) = state.save() {
11797        tracing::warn!("could not persist cache resource block: {error:#}");
11798    }
11799}
11800
11801/// Run `body` — a verify command batch — while holding the shared build
11802/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11803/// never interleave with another borrower's build against the same
11804/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11805/// timeout, or a human's own `magi review`. See the `cache` module doc for
11806/// why this matters more than Cargo's own per-target locking covers — two
11807/// *different* worktrees building the same package name/version into one
11808/// cache directory is a staleness bug, not a lock contention one.
11809///
11810/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11811/// agent waves do. Repository scheduling also has its own bounded,
11812/// cancellable resource-wait policy. Contention must not consume the
11813/// verification budget: `body` receives the full `budget` after both leases
11814/// are acquired and the cache is fresh. Without a shared cache, only
11815/// repository coordination is acquired before handing over the full budget.
11816///
11817/// A lease that cannot be acquired within the resource wait is reported as a single
11818/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11819/// verification — the same shape a spawn failure already takes in
11820/// [`run_commands`], so a caller need not special-case it.
11821#[allow(clippy::too_many_arguments)]
11822async fn with_cache_lease<'s, F, Fut>(
11823    state: &'s mut RunState,
11824    cache_dir: Option<&Path>,
11825    node: &str,
11826    seat: &str,
11827    worktree: &Path,
11828    head: &str,
11829    budget: Duration,
11830    context: &str,
11831    wait: &crate::repo_lock::WaitPolicy,
11832    body: F,
11833) -> Result<(Vec<CommandOutcome>, bool)>
11834where
11835    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11836    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11837{
11838    // Match editing waves: cache first, repository second. In particular,
11839    // pre_gate must never hold the repository while waiting for a wave's cache.
11840    let started = Instant::now();
11841    let home = crate::run::home();
11842    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11843    let guard = if let Some(cache_dir) = cache_dir {
11844        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11845        match acquire_cache_lease(
11846            state,
11847            cache_dir,
11848            &owner,
11849            resource_limit,
11850            node,
11851            context,
11852            &|| wait.cancelled(),
11853        )
11854        .await
11855        .and_then(|g| {
11856            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11857        }) {
11858            Ok(g) => Some(g),
11859            Err(e) => {
11860                record_cache_error(state, node, &e);
11861                return Ok((
11862                    vec![CommandOutcome {
11863                        command: "(waiting for the shared build cache)".into(),
11864                        code: None,
11865                        output_tail: e.to_string(),
11866                        duration_ms: started.elapsed().as_millis() as u64,
11867                        resource_blocked: true,
11868                    }],
11869                    false,
11870                ));
11871            }
11872        }
11873    } else {
11874        None
11875    };
11876    let _coordination = coordinate_operation(state, worktree, wait).await?;
11877    if let Some(cache_dir) = cache_dir {
11878        let identity = crate::cache::Identity::new(worktree, head);
11879        if let Err(e) = crate::cache::ensure_fresh(
11880            &home,
11881            cache_dir,
11882            &identity,
11883            &state.config.verify.cargo_manifests(),
11884        ) {
11885            record_cache_error(state, node, &e);
11886            state.event(
11887                "verify",
11888                format!(
11889                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11890                    worktree.display(),
11891                    short(head)
11892                ),
11893            );
11894            if let Some(guard) = guard {
11895                guard.release();
11896            }
11897            return Ok((
11898                vec![CommandOutcome {
11899                    command: "(confirming the shared build cache is fresh)".into(),
11900                    code: None,
11901                    output_tail: e.to_string(),
11902                    duration_ms: started.elapsed().as_millis() as u64,
11903                    resource_blocked: true,
11904                }],
11905                false,
11906            ));
11907        }
11908    }
11909    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11910    if !timed_out_pids.is_empty() {
11911        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11912    }
11913    // Drop repository coordination before releasing the cache, reversing the
11914    // acquisition order. A cancelled/error future also drops both guards.
11915    drop(_coordination);
11916    if let Some(guard) = guard {
11917        guard.release();
11918    }
11919    Ok((outcomes, retried))
11920}
11921
11922/// Poll `pids` — commands [`run_commands`] reports as still running when its
11923/// own timeout elapsed — until every one is confirmed gone, or
11924/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11925///
11926/// Real confirmation where confirmation is possible, not a substitute for
11927/// full process-tree observation: a grandchild the timed-out process spawned
11928/// and that survives independently of it is invisible to a pid check the
11929/// same way it always was, and continuing to observe and collect *that*
11930/// stays a different piece of work with its own owner. This only narrows a
11931/// fixed blind wait into an actual check of the pids this process does know
11932/// about.
11933async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11934    wait_for_pids_with(
11935        pids,
11936        crate::proc::pid_alive,
11937        LEASE_RELEASE_POLL,
11938        LEASE_RELEASE_MAX_WAIT,
11939    )
11940    .await;
11941}
11942
11943/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
11944/// interval and ceiling supplied by the caller, so the polling *logic* -
11945/// returns as soon as every pid reports dead, gives up at the ceiling
11946/// otherwise - is testable on millisecond durations without asking the real
11947/// OS about a pid at all.
11948async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11949    pids: &[u32],
11950    alive: F,
11951    poll: Duration,
11952    max_wait: Duration,
11953) {
11954    let deadline = Instant::now() + max_wait;
11955    loop {
11956        if pids.iter().all(|&pid| !alive(pid)) {
11957            return;
11958        }
11959        if Instant::now() >= deadline {
11960            return;
11961        }
11962        tokio::time::sleep(poll).await;
11963    }
11964}
11965
11966/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
11967/// admission that it could not even get a verify command to run, as opposed
11968/// to evidence the command actually produced? A caller that would otherwise
11969/// read a resource-blocked outcome as a red command must check this first:
11970/// see [`Runner::gate`], which retries rather than records `Blocked` when
11971/// this is true.
11972fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11973    outcomes.iter().any(|o| o.resource_blocked)
11974}
11975
11976/// What [`Runner::gate_fix_round`] decided.
11977pub(crate) enum GateFix {
11978    /// The tree changed and `verify.e2e` is still green: run the gate again.
11979    Retry,
11980    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
11981    /// last failure stands and the run ends blocked.
11982    Stop,
11983    /// `verify.e2e` could not run after the fix (magi's own contention):
11984    /// decide nothing now, a later reentry retries.
11985    Defer,
11986}
11987
11988/// Is every red command in `outcomes` an ordinary failure the code could
11989/// explain: it ran, exited non-zero, and said something?
11990///
11991/// A timeout, a spawn failure and a killed process all leave `code` `None`;
11992/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
11993/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
11994/// what the command is stays the gate's business.
11995pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11996    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11997    red.peek().is_some()
11998        && red.all(|o| {
11999            !o.resource_blocked
12000                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12001                && !o.output_tail.trim().is_empty()
12002        })
12003}
12004
12005/// Describe one verify command's outcome for the event log, distinguishing a
12006/// build/link failure — the toolchain never produced a binary to run — from
12007/// an actual test failure, since only the latter is a verdict on the patch.
12008fn e2e_outcome_label(o: &CommandOutcome) -> String {
12009    if o.ok() {
12010        return "pass".to_owned();
12011    }
12012    let reason = if o.build_failed() {
12013        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12014    } else {
12015        format!("FAIL ({:?})", o.code)
12016    };
12017    format!(
12018        "{reason}\n{}",
12019        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12020    )
12021}
12022
12023/// Run `verify.e2e`, retrying once if the first attempt could not build or
12024/// link — a build/link failure is frequently a race against a shared
12025/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
12026/// `verify` event per command, tagged with `context` (normally `"round N"`)
12027/// so the two call sites that need this — the ordinary per-round leg in
12028/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
12029/// it will ever call a round green — read identically in the event log.
12030async fn run_e2e_with_retry(
12031    state: &mut RunState,
12032    shell: &[String],
12033    commands: &[String],
12034    worktree: &Path,
12035    timeout: Duration,
12036    context: &str,
12037) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12038    let (mut e2e, mut timed_out_pids) = run_commands(
12039        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12040    )
12041    .await;
12042    for o in &e2e {
12043        state.event(
12044            "verify",
12045            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12046        );
12047    }
12048    // A build/link failure is not a verdict on the patch — it is frequently a
12049    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
12050    // one retry before letting a red like that decide the round.
12051    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12052    if verify_retried {
12053        state.event(
12054            "verify",
12055            format!(
12056                "{context}: verify could not build/link, not a test result — retrying once \
12057                 before concluding"
12058            ),
12059        );
12060        let retried = run_commands(
12061            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12062        )
12063        .await;
12064        e2e = retried.0;
12065        // Both attempts' timeouts matter, not just the last one: the first
12066        // attempt's descendants may still be alive alongside the retry's.
12067        timed_out_pids.extend(retried.1);
12068        for o in &e2e {
12069            state.event(
12070                "verify",
12071                format!(
12072                    "{context}: retry `{}` -> {}",
12073                    o.command,
12074                    e2e_outcome_label(o)
12075                ),
12076            );
12077        }
12078    }
12079    (e2e, verify_retried, timed_out_pids)
12080}
12081
12082/// Run configured shell commands in `cwd`, in order. The second element is
12083/// the pid of every command that hit `timeout` and was still running when
12084/// this stopped waiting on it (best-effort: `None` when the platform did not
12085/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
12086/// that releases a shared resource afterward needs to know.
12087///
12088/// Records `task` into [`RunState::active`] at every command boundary
12089/// (`RunState::task_command`) and clears it once the whole list has run
12090/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
12091/// for minutes with no seat and no output of its own to show for it (see
12092/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
12093/// apart from "not yet run" without this), and this is the only place that
12094/// knows which command is running right now and how many are left. Three
12095/// saves per command — start, not per second — matching the same "only at a
12096/// boundary" rule [`wave`] already follows for seats.
12097#[allow(clippy::too_many_arguments)]
12098async fn run_commands(
12099    state: &mut RunState,
12100    node: &str,
12101    task: &str,
12102    attempt: usize,
12103    shell: &[String],
12104    commands: &[String],
12105    cwd: &Path,
12106    timeout: Duration,
12107) -> (Vec<CommandOutcome>, Vec<u32>) {
12108    if commands.is_empty() {
12109        // Nothing to mark as running and nothing to clear — an empty list
12110        // means "not configured", and touching `active` (or the disk) over
12111        // that would be a write for every round of a repo with no
12112        // `verify.e2e` / `verify.gate` commands at all.
12113        return (Vec::new(), Vec::new());
12114    }
12115    let mut out = Vec::new();
12116    let mut timed_out_pids = Vec::new();
12117    let total = commands.len();
12118    for (idx, command) in commands.iter().enumerate() {
12119        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12120        if let Err(e) = state.save() {
12121            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12122        }
12123        let started = Instant::now();
12124        let mut cmd = tokio::process::Command::new(&shell[0]);
12125        cmd.quiet();
12126        // Verify the checkout as a standalone caller. Worker ownership and
12127        // pinned executable selection belong to this driver, not to tests or
12128        // tools it runs. Preserve home/seat context and Running's containment.
12129        for key in crate::worker::INVOCATION_ENV {
12130            cmd.env_remove(key);
12131        }
12132        cmd.args(&shell[1..])
12133            .arg(command)
12134            .current_dir(cwd)
12135            .stdin(std::process::Stdio::null())
12136            .stdout(std::process::Stdio::piped())
12137            .stderr(std::process::Stdio::piped())
12138            .kill_on_drop(true);
12139        let mut outputs = Vec::new();
12140        let protected = (|| -> Result<()> {
12141            crate::incremental::configure(&mut cmd, cwd)?;
12142            let mut paths = vec![cwd.to_owned()];
12143            if let Some(cache) = state.config.cache_dir() {
12144                paths.push(cache);
12145            }
12146            outputs.extend(crate::retention::OutputLease::admit(
12147                &crate::run::home(),
12148                &paths,
12149                state.config.disk.min_free_bytes,
12150            )?);
12151            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12152                outputs.push(guard);
12153            }
12154            if let Some(cache) = state.config.cache_dir() {
12155                outputs.push(crate::retention::OutputLease::cache(
12156                    &crate::run::home(),
12157                    &cache,
12158                    false,
12159                )?);
12160            }
12161            Ok(())
12162        })();
12163        if let Err(e) = protected {
12164            let detail = format!("execution admission unavailable: {e:#}");
12165            state.resource_block = Some(crate::run::ResourceBlock {
12166                cause: crate::run::ResourceBlockCause::Other,
12167                node: node.to_owned(),
12168                kind: if detail.contains("disk admission") {
12169                    "disk admission"
12170                } else {
12171                    "output ownership"
12172                }
12173                .to_owned(),
12174                detail: detail.clone(),
12175                at: Timestamp::now(),
12176            });
12177            out.push(CommandOutcome {
12178                command: command.clone(),
12179                code: None,
12180                output_tail: detail,
12181                duration_ms: 0,
12182                resource_blocked: true,
12183            });
12184            break;
12185        }
12186        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12187        let (code, body) = match spawned {
12188            Ok(mut running) => {
12189                let pid = running.child().id();
12190                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12191                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12192                let mut paths = vec![cwd.to_owned()];
12193                if let Some(cache) = state.config.cache_dir() {
12194                    paths.push(cache);
12195                }
12196                let mut pressure = None;
12197                let status = tokio::select! {
12198                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12199                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12200                        pressure = Some(reason); None
12201                    }
12202                };
12203                let (code, failure) = match status {
12204                    Some(Ok(Ok(status))) => (
12205                        status.code(),
12206                        status
12207                            .code()
12208                            .is_none()
12209                            .then(|| format!("command terminated: {status}")),
12210                    ),
12211                    Some(Ok(Err(error))) => {
12212                        (None, Some(format!("failed waiting for command: {error}")))
12213                    }
12214                    Some(Err(_)) => {
12215                        // Leased cleanup confirms process-tree death itself.
12216                        // Unleased commands still need the caller's PID check
12217                        // after kill-on-drop, before releasing cache ownership.
12218                        if !running.coordinated()
12219                            && let Some(pid) = pid
12220                        {
12221                            timed_out_pids.push(pid);
12222                        }
12223                        (
12224                            None,
12225                            Some(format!("timed out after {}s", timeout.as_secs())),
12226                        )
12227                    }
12228                    None => (None, None),
12229                };
12230                running.finish().await;
12231                if let Some(reason) = pressure {
12232                    state.resource_block = Some(crate::run::ResourceBlock {
12233                        cause: crate::run::ResourceBlockCause::Other,
12234                        node: node.to_owned(),
12235                        kind: "disk pressure".to_owned(),
12236                        detail: reason.clone(),
12237                        at: Timestamp::now(),
12238                    });
12239                    out.push(CommandOutcome {
12240                        command: command.clone(),
12241                        code: None,
12242                        output_tail: reason,
12243                        duration_ms: started.elapsed().as_millis() as u64,
12244                        resource_blocked: true,
12245                    });
12246                    break;
12247                }
12248                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12249                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12250                if let Some(failure) = failure {
12251                    body.push('\n');
12252                    body.push_str(&failure);
12253                }
12254                (code, body)
12255            }
12256            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12257        };
12258        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12259        if !body.is_empty() {
12260            // The full text is kept whatever the summary dropped. A fresh
12261            // name per command so a resume or retry never overwrites an
12262            // earlier run's output. A failed write changes no verdict.
12263            match crate::run::write_verification_artifact(state, task, &body) {
12264                Ok(name) if summary != body => {
12265                    summary = format!("[full output: artifacts/{name}]\n{summary}");
12266                }
12267                Ok(_) => {}
12268                Err(e) => {
12269                    let warning = format!("could not save the full {task} output: {e:#}");
12270                    tracing::warn!("{warning}");
12271                    state.event(node, warning);
12272                    if summary != body {
12273                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12274                    }
12275                }
12276            }
12277        }
12278        let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12279        out.push(CommandOutcome {
12280            command: command.clone(),
12281            code,
12282            output_tail: summary,
12283            duration_ms: started.elapsed().as_millis() as u64,
12284            resource_blocked,
12285        });
12286        if resource_blocked {
12287            state.resource_block = Some(crate::run::ResourceBlock {
12288                cause: crate::run::ResourceBlockCause::Other,
12289                node: node.to_owned(),
12290                kind: "incremental cache admission".into(),
12291                detail: body,
12292                at: Timestamp::now(),
12293            });
12294            break;
12295        }
12296    }
12297    state.task_finished(task);
12298    if let Err(e) = state.save() {
12299        tracing::warn!("could not persist the end of {task}: {e:#}");
12300    }
12301    (out, timed_out_pids)
12302}
12303
12304/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
12305/// section (`report::run`) and in the `merge` event this node records — for
12306/// the operator to run by hand.
12307///
12308/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
12309/// branch whose ruleset forbids merge commits (GitHub's "must not contain
12310/// merge commits", or "require linear history") rejects the push a `--no-ff`
12311/// merge would produce, which is exactly the guidance this function replaces.
12312/// `message`'s first line becomes the squash commit's subject, matching the
12313/// note `report::run` prints alongside this command — see that function for
12314/// why an explicit subject is not optional there.
12315fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12316    let repo = repo.display();
12317    match style {
12318        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12319        MergeStyle::Squash => {
12320            // The subject sits inside double quotes, and a title an agent
12321            // wrote may carry the characters that break out of them.
12322            let subject = message
12323                .lines()
12324                .next()
12325                .unwrap_or(branch)
12326                .replace(['\\', '"', '$', '`'], "");
12327            format!(
12328                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12329            )
12330        }
12331        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12332    }
12333}
12334
12335/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
12336/// under the hood, rejects a `title` over 256 characters and the whole
12337/// command fails — no PR at all, for a run whose body was otherwise fine
12338/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
12339/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
12340/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
12341/// this bound. It is a margin, not a guarantee — a title packed
12342/// with multi-unit characters could still in principle land close to the
12343/// edge, but a real task title's occasional emoji or accented letter fits
12344/// comfortably inside it.
12345const PR_TITLE_MAX: usize = 240;
12346
12347/// A pull request title from the opening line of `text`, or `None` when that
12348/// line is not English (GitHub text is) or has no letters.
12349///
12350/// A line within `max` is kept as is. A longer one is cut at the end of its
12351/// first sentence when that falls inside `max`, else at a word boundary with a
12352/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
12353/// make every merely-truncated title look non-English). The language check
12354/// runs on the kept text before any mark is added, so only what GitHub will
12355/// show is judged: an English opening followed by non-ASCII far past the cut
12356/// still passes.
12357fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12358    let line = queue::first_line(text)?;
12359    let chars: Vec<char> = line.chars().collect();
12360    let (kept, mark) = if chars.len() <= max {
12361        (line.to_owned(), "")
12362    } else if let Some(end) = sentence_end(&chars, max) {
12363        (chars[..end].iter().collect::<String>(), "")
12364    } else {
12365        let room = max.saturating_sub(3);
12366        // Cut at the last space inside the room; when the char just past the
12367        // room is a space the room already ends on a word.
12368        let cut = if chars[room].is_whitespace() {
12369            room
12370        } else {
12371            chars[..room]
12372                .iter()
12373                .rposition(|c| c.is_whitespace())
12374                .unwrap_or(room)
12375        };
12376        let head: String = chars[..cut].iter().collect();
12377        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12378        (head.to_owned(), "...")
12379    };
12380    if kept.is_empty() {
12381        return None;
12382    }
12383    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12384        return None;
12385    }
12386    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12387        return None;
12388    }
12389    Some(format!("{kept}{mark}"))
12390}
12391
12392/// The char length of the first sentence of `chars` when it ends within `max`
12393/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
12394/// abbreviations so `e.g. foo` does not end a title early.
12395fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12396    const MIN: usize = 20;
12397    for i in MIN..max.min(chars.len()) {
12398        if !matches!(chars[i], '.' | '!' | '?') {
12399            continue;
12400        }
12401        let Some(&next) = chars.get(i + 1) else {
12402            continue;
12403        };
12404        if !next.is_whitespace() {
12405            continue;
12406        }
12407        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12408        if after.is_some_and(|c| c.is_ascii_lowercase()) {
12409            continue;
12410        }
12411        let word: String = chars[..i]
12412            .iter()
12413            .rev()
12414            .take_while(|c| !c.is_whitespace())
12415            .collect::<Vec<_>>()
12416            .into_iter()
12417            .rev()
12418            .collect();
12419        let word = word.to_ascii_lowercase();
12420        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12421            continue;
12422        }
12423        let end = chars[..i]
12424            .iter()
12425            .rposition(|c| !c.is_whitespace())
12426            .map_or(i, |p| p + 1);
12427        return Some(end);
12428    }
12429    None
12430}
12431
12432/// What `merge = "pr"` (and the merge commit of the other modes) says about a
12433/// change: a title and a body describing what was *implemented*, not the task
12434/// that asked for it. A task reads as a request; a reader of the merged
12435/// history wants the change.
12436struct PrMessage {
12437    title: String,
12438    body: String,
12439}
12440
12441impl PrMessage {
12442    /// Title, blank line, body. The first line is the squash/merge commit
12443    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
12444    /// has to stay one sensible line.
12445    fn commit_message(&self) -> String {
12446        format!("{}\n\n{}", self.title, self.body)
12447    }
12448}
12449
12450/// The text after a leading `TITLE:` (any case) on `line`.
12451fn title_marker(line: &str) -> Option<&str> {
12452    let line = line.trim();
12453    let head = line.get(..6)?;
12454    head.eq_ignore_ascii_case("title:")
12455        .then(|| line[6..].trim())
12456}
12457
12458/// The implementer's own one-line title: the `TITLE:` line the implement
12459/// prompt asks for at the top of its SUMMARY. Candidate commits are all
12460/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
12461/// source, and a title that says as much is refused here too.
12462fn summary_title(summary: &str) -> Option<String> {
12463    let first = summary.lines().find(|l| !l.trim().is_empty())?;
12464    let raw = title_marker(first)?;
12465    if raw.is_empty() {
12466        return None;
12467    }
12468    let title = queue::title_from(raw, PR_TITLE_MAX);
12469    let lower = title.to_ascii_lowercase();
12470    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12471        return None;
12472    }
12473    Some(title)
12474}
12475
12476/// How `open_review`'s instruction begins; see [`landing_title`].
12477const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12478
12479/// Marker `open_review` gives a candidate that nothing in the roster wrote.
12480pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12481
12482/// Does this run review work that already existed, rather than implement a
12483/// task? Runs recorded before `reviewed_commits` existed carry only the
12484/// candidate marker.
12485pub(crate) fn is_review_run(state: &RunState) -> bool {
12486    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12487}
12488
12489/// The title of a review-only run: the subject of the oldest commit under
12490/// review. Later commits are usually fixups, and `instruction` is the review
12491/// prompt, which says nothing about the change. GitHub text is English, so a
12492/// non-ASCII or blank subject yields `None` and the caller's neutral title.
12493fn review_title(state: &RunState) -> Option<String> {
12494    english_subject(
12495        state.reviewed_commits.as_ref()?.first()?,
12496        state.config.graph.github_english(),
12497    )
12498}
12499
12500/// `raw` as a pull request title, or `None` when it is blank, not English
12501/// (GitHub text is), or one of magi's own candidate commit subjects.
12502fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12503    let raw = raw.trim();
12504    if raw.is_empty() {
12505        return None;
12506    }
12507    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12508        return None;
12509    }
12510    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12511        return None;
12512    }
12513    let title = queue::title_from(raw, PR_TITLE_MAX);
12514    let lower = title.to_ascii_lowercase();
12515    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12516        return None;
12517    }
12518    Some(title)
12519}
12520
12521/// What a review-only run's branch says about itself, read at the moment the
12522/// pull request is opened.
12523#[derive(Debug, Clone, PartialEq, Eq)]
12524struct BranchFacts {
12525    /// `(subject, body)` of each commit, oldest first.
12526    commits: Vec<(String, String)>,
12527    /// Trimmed `git diff --stat`.
12528    stat: String,
12529}
12530
12531/// Longest diff stat shown: this many file lines plus the summary line.
12532const STAT_FILE_LINES: usize = 25;
12533/// Cap on the commit list, well inside GitHub's 65536-character body limit.
12534const COMMITS_MAX_CHARS: usize = 20_000;
12535
12536/// Read the commits and diff stat of `base..branch`. `None` when git cannot
12537/// say or finds nothing, so the caller falls back to what the run recorded.
12538async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12539    let commits = git::commit_log(repo, base, branch).await.ok()?;
12540    if commits.is_empty() {
12541        return None;
12542    }
12543    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12544    let lines: Vec<&str> = stat.lines().collect();
12545    let stat = if lines.len() > STAT_FILE_LINES + 1 {
12546        let omitted = lines.len() - 1 - STAT_FILE_LINES;
12547        let more = format!(" ... {omitted} more file(s)");
12548        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12549        kept.push(&more);
12550        kept.push(lines[lines.len() - 1]);
12551        kept.join("\n")
12552    } else {
12553        lines.join("\n")
12554    };
12555    Some(BranchFacts { commits, stat })
12556}
12557
12558/// Defang what would break the surrounding markdown: a closing `</details>`
12559/// and a code fence.
12560fn markdown_safe(text: &str) -> String {
12561    text.replace("</details>", "&lt;/details&gt;")
12562        .replace("\x60\x60\x60", "~~~")
12563}
12564
12565fn neutral_title(state: &RunState, winner: char) -> String {
12566    format!(
12567        "chore: land candidate {} of run {}",
12568        winner.to_ascii_uppercase(),
12569        state.id
12570    )
12571}
12572
12573/// Reuse the graph's bounded fixer without starting another graph or driver.
12574/// Copy the working state back even on errors so spent rounds remain visible.
12575pub(crate) async fn repair_landing_gate(
12576    state: &mut RunState,
12577    winner: &Candidate,
12578    outcomes: &[CommandOutcome],
12579) -> Result<GateFix> {
12580    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12581        return Ok(GateFix::Stop);
12582    }
12583    let mut runner = Runner {
12584        roles: state.config.resolve_roles()?,
12585        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12586        state: state.clone(),
12587        pause: Pause::new(),
12588        interrupt: Pause::new(),
12589    };
12590    let result = runner.gate_fix_round(winner, outcomes).await;
12591    *state = runner.state;
12592    result
12593}
12594
12595/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
12596/// A completed failure is terminal on that SHA. Resource waits remain retryable.
12597pub(crate) async fn verify_current_head(
12598    state: &mut RunState,
12599    winner: &Candidate,
12600    wait: &crate::repo_lock::WaitPolicy,
12601) -> Result<bool> {
12602    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12603    // Preserve the previous record before replacing it, including on crash recovery.
12604    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12605        state
12606            .verification_history
12607            .push(crate::run::VerificationEvidence {
12608                superseded_by: Some(head.clone()),
12609                archived_at: Some(Timestamp::now()),
12610                fixes_spent: Some(state.gate_fixes.len()),
12611                gate_head: None,
12612                gate_ran: false,
12613                gate: Vec::new(),
12614                e2e: state.head_e2e.take(),
12615            });
12616    }
12617    if let Some(e) = &state.head_e2e
12618        && e.head == head
12619    {
12620        if e.passed {
12621            return Ok(true);
12622        }
12623        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12624            state.status = RunStatus::Blocked;
12625            state.save()?;
12626            return Ok(false);
12627        }
12628    } else if state.reviews.last().is_some_and(|r| {
12629        r.verified_head.as_deref() == Some(head.as_str())
12630            && (r.e2e_status() == E2eStatus::Passed
12631                || (r.e2e_status() == E2eStatus::NotConfigured
12632                    && state.config.verify.e2e.is_empty()))
12633    }) || state
12634        .gate_repair_e2e
12635        .as_ref()
12636        .is_some_and(|e| e.head == head && e.passed)
12637    {
12638        return Ok(true);
12639    }
12640    if let Some(e) = &state.gate_repair_e2e
12641        && e.head == head
12642        && !e.passed
12643        && !e.outcomes.is_empty()
12644        && !verify_inconclusive(&e.outcomes)
12645    {
12646        state.status = RunStatus::Blocked;
12647        state.save()?;
12648        return Ok(false);
12649    }
12650    state.head_e2e = Some(crate::run::GateRepairE2e {
12651        head: head.clone(),
12652        passed: false,
12653        outcomes: Vec::new(),
12654    });
12655    state.save()?;
12656    let commands = state.config.verify.e2e.clone();
12657    let shell = state.config.shell();
12658    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12659    let cache_dir = state.config.cache_dir();
12660    let outcomes = if commands.is_empty() {
12661        Vec::new()
12662    } else {
12663        let (outcomes, _) = with_cache_lease(
12664            state,
12665            cache_dir.as_deref(),
12666            "e2e",
12667            "e2e",
12668            &winner.worktree,
12669            &head,
12670            timeout,
12671            "current head e2e",
12672            wait,
12673            |state, budget| async move {
12674                run_e2e_with_retry(
12675                    state,
12676                    &shell,
12677                    &commands,
12678                    &winner.worktree,
12679                    budget,
12680                    "current head e2e",
12681                )
12682                .await
12683            },
12684        )
12685        .await?;
12686        outcomes
12687    };
12688    let unresolved = verify_inconclusive(&outcomes);
12689    let passed = !unresolved
12690        && outcomes.iter().all(CommandOutcome::ok)
12691        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12692    if !outcomes.is_empty() && !unresolved {
12693        state.record_base_progress(&head);
12694    }
12695    state.head_e2e = Some(crate::run::GateRepairE2e {
12696        head: head.clone(),
12697        passed,
12698        outcomes,
12699    });
12700    if !passed && !unresolved {
12701        state.status = RunStatus::Blocked;
12702        state.event(
12703            "e2e",
12704            format!("verify.e2e failed on head {head}; nothing published"),
12705        );
12706    }
12707    state.save()?;
12708    Ok(passed)
12709}
12710
12711/// Recheck a repair's e2e requirement across failures, resource deferrals and
12712/// process interruptions. A successful result only applies to its exact SHA.
12713pub(crate) async fn check_gate_repair_e2e(
12714    state: &mut RunState,
12715    winner: &Candidate,
12716    wait: &crate::repo_lock::WaitPolicy,
12717) -> Result<GateFix> {
12718    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12719        return Ok(GateFix::Retry);
12720    }
12721    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12722    if state
12723        .gate_repair_e2e
12724        .as_ref()
12725        .is_some_and(|r| r.head == head && r.passed)
12726    {
12727        return Ok(GateFix::Retry);
12728    }
12729    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12730        head: head.clone(),
12731        passed: false,
12732        outcomes: Vec::new(),
12733    });
12734    state.save()?;
12735    let commands = state.config.verify.e2e.clone();
12736    let outcomes = if commands.is_empty() {
12737        Vec::new()
12738    } else {
12739        let shell = state.config.shell();
12740        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12741        let cache_dir = state.config.cache_dir();
12742        let (outcomes, _) = with_cache_lease(
12743            state,
12744            cache_dir.as_deref(),
12745            "e2e",
12746            "e2e",
12747            &winner.worktree,
12748            &head,
12749            timeout,
12750            "gate repair e2e",
12751            wait,
12752            |state, budget| async move {
12753                run_e2e_with_retry(
12754                    state,
12755                    &shell,
12756                    &commands,
12757                    &winner.worktree,
12758                    budget,
12759                    "gate repair e2e",
12760                )
12761                .await
12762            },
12763        )
12764        .await?;
12765        outcomes
12766    };
12767    let deferred = verify_inconclusive(&outcomes);
12768    let passed = outcomes.iter().all(CommandOutcome::ok)
12769        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12770    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12771        head,
12772        passed,
12773        outcomes,
12774    });
12775    state.save()?;
12776    if deferred {
12777        Ok(GateFix::Defer)
12778    } else if !passed {
12779        state.event(
12780            "gate",
12781            "verify.e2e failed after gate repair; nothing published",
12782        );
12783        Ok(GateFix::Stop)
12784    } else {
12785        Ok(GateFix::Retry)
12786    }
12787}
12788
12789/// Run the same configured gate for a landing head and a fresh graph.
12790pub(crate) async fn run_landing_gate(
12791    state: &mut RunState,
12792    winner: &Candidate,
12793    wait: &crate::repo_lock::WaitPolicy,
12794) -> Result<Vec<CommandOutcome>> {
12795    let shell = state.config.shell();
12796    let gate_commands = state.config.verify.gate.clone();
12797    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12798    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12799        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12800        Vec::new()
12801    } else {
12802        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12803        let cache_dir = state.config.cache_dir();
12804        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12805        let (outcomes, _) = with_cache_lease(
12806            state,
12807            cache_dir.as_deref(),
12808            "gate",
12809            "gate",
12810            &winner.worktree,
12811            &head,
12812            timeout,
12813            "final gate",
12814            wait,
12815            |state, budget| {
12816                let shell = shell.clone();
12817                let gate_commands = gate_commands.clone();
12818                let winner = winner.clone();
12819                async move {
12820                    Runner::run_pre_gate(state, &winner).await;
12821                    // Bind command evidence after mechanical fixes, before commands start.
12822                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12823                    let (outcomes, timed_out_pids) = run_commands(
12824                        state,
12825                        "gate",
12826                        "gate",
12827                        0,
12828                        &shell,
12829                        &gate_commands,
12830                        &winner.worktree,
12831                        budget,
12832                    )
12833                    .await;
12834                    (outcomes, false, timed_out_pids)
12835                }
12836            },
12837        )
12838        .await?;
12839        outcomes
12840    };
12841    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12842    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12843        let mut outcomes = outcomes;
12844        outcomes.push(CommandOutcome {
12845            command: "gate head stability".into(), code: None,
12846            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12847            duration_ms: 0, resource_blocked: false,
12848        });
12849        return Ok(outcomes);
12850    }
12851    if outcomes.is_empty() {
12852        // Nothing configured to check — distinct from every other
12853        // silence in this run's event log, since an empty `gate` alone
12854        // no longer says whether the gate ran at all (see
12855        // `RunState::gate_ran`'s own doc).
12856        state.event("gate", "no gate commands configured; not tested");
12857    }
12858    for o in &outcomes {
12859        state.event(
12860            "gate",
12861            format!(
12862                "`{}` -> {}",
12863                o.command,
12864                if o.ok() {
12865                    "pass".to_owned()
12866                } else {
12867                    format!(
12868                        "FAIL ({:?})\n{}",
12869                        o.code,
12870                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12871                    )
12872                }
12873            ),
12874        );
12875    }
12876    Ok(outcomes)
12877}
12878
12879/// The pull request title to hand to `land::merge_subject`. A review-only run
12880/// opened by an earlier build titled its pull request with the review prompt;
12881/// that title is dropped (empty, so the fallback applies) rather than landed.
12882/// Any other title, including an operator's rename, passes through untouched,
12883/// and so does every title of a run that implements a task.
12884pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12885    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12886        ""
12887    } else {
12888        pr_title
12889    }
12890}
12891
12892/// What the squash subject falls back to when the pull request title is empty
12893/// or candidate-shaped: for a review-only run the derived title, never the
12894/// review prompt held in `instruction`.
12895pub fn landing_subject_source(state: &RunState) -> String {
12896    if is_review_run(state) {
12897        let winner = state.candidates.first().map_or('A', |c| c.label);
12898        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12899    }
12900    state.instruction.clone()
12901}
12902
12903/// `summary` without its `TITLE:` line, which the pull request title already
12904/// carries.
12905fn summary_without_title(summary: &str) -> String {
12906    let mut lines = summary.trim().lines().peekable();
12907    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12908        lines.next();
12909    }
12910    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12911}
12912
12913/// The pull request title and body for the winning candidate.
12914///
12915/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
12916/// the task's own opening line via [`queue::title_from`] when there is none.
12917/// `state.instruction` can open with blank lines (`task_text` only rejects a
12918/// body that is blank *entirely*), which `title_from` skips.
12919///
12920/// Publishing uses an independent description generated from the final diff
12921/// and structured evidence. Internal summaries and reports remain local.
12922/// Non-publishing previews may display the original task and candidate summary.
12923/// The footer retains the run and candidate provenance.
12924#[cfg(test)]
12925fn pr_message(state: &RunState, winner: char) -> PrMessage {
12926    pr_message_with(state, winner, None)
12927}
12928
12929/// [`pr_message`] with what the branch of a review-only run says about itself.
12930/// `facts` is ignored for a run that implements a task.
12931#[cfg(test)]
12932fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12933    let raw = pr_message_raw(
12934        state,
12935        winner,
12936        facts,
12937        crate::pr_outcome::FinalHead::Unreported,
12938        ChangeNote::NotAsked,
12939    );
12940    let id = crate::scrub::Identity::current();
12941    PrMessage {
12942        title: crate::scrub::scrub(&raw.title, &id),
12943        body: crate::scrub::scrub(&raw.body, &id),
12944    }
12945}
12946
12947/// What the public description may say about the change itself.
12948#[derive(Debug, Clone, Copy)]
12949enum ChangeNote<'a> {
12950    /// A local preview or non-publishing merge mode: no public generation needed.
12951    NotAsked,
12952    /// A summary written from the final diff.
12953    Summary(&'a crate::run::ChangeSummary),
12954    /// A summary was asked for and could not be produced; the text says why.
12955    Unavailable(&'a str),
12956}
12957
12958fn pr_message_raw(
12959    state: &RunState,
12960    winner: char,
12961    facts: Option<&BranchFacts>,
12962    head: crate::pr_outcome::FinalHead<'_>,
12963    change: ChangeNote<'_>,
12964) -> PrMessage {
12965    if !matches!(change, ChangeNote::NotAsked) {
12966        let (title, mut body) = match change {
12967            ChangeNote::Summary(c) => (
12968                c.title.clone(),
12969                format!("## Summary\n\n{}\n\n", c.body.trim()),
12970            ),
12971            ChangeNote::Unavailable(why) => (
12972                neutral_title(state, winner),
12973                format!(
12974                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12975                ),
12976            ),
12977            ChangeNote::NotAsked => unreachable!(),
12978        };
12979        body.push_str(&public_pr_evidence(state, head));
12980        body.push_str(&format!(
12981            "\n---\nmagi:run/{} magi:candidate-{}\n",
12982            state.id,
12983            winner.to_ascii_lowercase()
12984        ));
12985        return PrMessage { title, body };
12986    }
12987    pr_preview(state, winner, facts, head)
12988}
12989
12990/// Append public facts from the run records after every prose generation,
12991/// including posting-gate rewrites. The agent cannot drop this provenance.
12992fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12993    let mut body = crate::pr_outcome::review_rounds(state);
12994    body.push_str(&crate::pr_outcome::final_verification(state, head));
12995    let open = state.last_round_findings();
12996    if !open.is_empty() {
12997        body.push_str("## Open review findings\n\n");
12998        for f in open {
12999            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13000        }
13001        body.push('\n');
13002    }
13003    if let Some(c) = state.change_summary.as_ref()
13004        && head == crate::pr_outcome::FinalHead::At(&c.head)
13005    {
13006        body.push_str(&format!(
13007            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13008            c.base, c.head
13009        ));
13010    }
13011    body
13012}
13013
13014/// Local-only preview of the operator's task and candidate summary.
13015fn pr_preview(
13016    state: &RunState,
13017    winner: char,
13018    facts: Option<&BranchFacts>,
13019    head: crate::pr_outcome::FinalHead<'_>,
13020) -> PrMessage {
13021    let summary = state
13022        .candidates
13023        .iter()
13024        .find(|c| c.label == winner)
13025        .map(|c| c.summary.as_str())
13026        .unwrap_or_default();
13027    // The fallback is the operator's own words and may not be English; GitHub
13028    // text always is, so a non-English task gets a neutral title instead.
13029    let review = is_review_run(state);
13030    let title = if review {
13031        facts
13032            .and_then(|f| {
13033                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13034            })
13035            .or_else(|| review_title(state))
13036            .or_else(|| {
13037                state
13038                    .candidates
13039                    .iter()
13040                    .find(|c| c.label == winner)
13041                    .filter(|c| !c.branch.starts_with("magi/"))
13042                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13043            })
13044            .unwrap_or_else(|| neutral_title(state, winner))
13045    } else {
13046        summary_title(summary).unwrap_or_else(|| {
13047            english_title(
13048                &state.instruction,
13049                PR_TITLE_MAX,
13050                state.config.graph.github_english(),
13051            )
13052            .unwrap_or_else(|| neutral_title(state, winner))
13053        })
13054    };
13055
13056    let mut body = String::new();
13057    let what = summary_without_title(summary);
13058    if !what.is_empty() {
13059        body.push_str("## Summary\n\n");
13060        body.push_str(&what);
13061        body.push_str("\n\n");
13062    }
13063
13064    // Review history uses structured facts across all rounds, never reports.
13065    body.push_str(&crate::pr_outcome::review_rounds(state));
13066    body.push_str(&crate::pr_outcome::final_verification(state, head));
13067    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13068
13069    let open = state.open_findings();
13070    if !open.is_empty() {
13071        body.push_str("## Open review findings\n\n");
13072        for f in &open {
13073            body.push_str(&format!(
13074                "- `{}` [{:?}] {}\n",
13075                f.id, f.severity, "Context retained in local review records."
13076            ));
13077        }
13078        body.push('\n');
13079    }
13080
13081    if let Some(fix) = fix
13082        && !fix.rejected.is_empty()
13083    {
13084        body.push_str("## Declined by the fixer\n\n");
13085        for r in &fix.rejected {
13086            body.push_str(&format!(
13087                "- `{}`: {}\n",
13088                r.id, "Reason retained in local review records."
13089            ));
13090        }
13091        body.push('\n');
13092    }
13093
13094    if review {
13095        // The review prompt is not the task; list what the branch carries.
13096        body.push_str("## Commits under review\n\n");
13097        if let Some(facts) = facts {
13098            let mut left = COMMITS_MAX_CHARS;
13099            for (i, (subject, text)) in facts.commits.iter().enumerate() {
13100                let mut entry = format!("- {}\n", markdown_safe(subject));
13101                for l in markdown_safe(text).lines() {
13102                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
13103                }
13104                if left == 0 {
13105                    body.push_str(&format!(
13106                        "- ... {} more commit(s)\n",
13107                        facts.commits.len() - i
13108                    ));
13109                    break;
13110                }
13111                if entry.len() > left {
13112                    // Even the first commit is cut: one huge body must not
13113                    // push the whole description past GitHub's limit.
13114                    let mut end = left;
13115                    while !entry.is_char_boundary(end) {
13116                        end -= 1;
13117                    }
13118                    entry.truncate(end);
13119                    entry.push_str("\n  ... (truncated)\n");
13120                    left = 0;
13121                } else {
13122                    left -= entry.len();
13123                }
13124                body.push_str(&entry);
13125            }
13126            if !facts.stat.trim().is_empty() {
13127                body.push_str(&format!(
13128                    "\n## Diff stat\n\n```\n{}\n```\n",
13129                    markdown_safe(facts.stat.trim())
13130                ));
13131            }
13132        } else {
13133            match &state.reviewed_commits {
13134                Some(subjects) => {
13135                    for s in subjects {
13136                        body.push_str(&format!("- {}\n", s.trim()));
13137                    }
13138                }
13139                None => {
13140                    // An older run kept only the prompt, with the commit list
13141                    // after its first paragraph.
13142                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13143                    body.push_str(rest.trim());
13144                    body.push('\n');
13145                }
13146            }
13147        }
13148    } else {
13149        let task = state.instruction.trim();
13150        let task = if task.is_empty() {
13151            "(empty task)"
13152        } else {
13153            task
13154        };
13155        body.push_str(&format!(
13156            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13157            task.replace("</details>", "&lt;/details&gt;")
13158        ));
13159    }
13160
13161    body.push_str(&format!(
13162        "\n---\nmagi:run/{} magi:candidate-{}\n",
13163        state.id,
13164        winner.to_ascii_lowercase()
13165    ));
13166
13167    PrMessage { title, body }
13168}
13169
13170/// Source imports have already been prepared from structured inputs. Legacy
13171/// inferred references must never instruct a seat to reconstruct those imports.
13172fn seeded_instruction(state: &RunState) -> String {
13173    let mut inputs = Vec::new();
13174    if let Some(start) = &state.sources.start_from {
13175        inputs.push(format!(
13176            "Start from `{}` at `{}` (including ancestry).",
13177            start.token, start.sha
13178        ));
13179    }
13180    for pick in &state.sources.pick_commits {
13181        inputs.push(format!(
13182            "Pick `{}` at `{}` (single commit).",
13183            pick.token, pick.sha
13184        ));
13185    }
13186    if inputs.is_empty() {
13187        state.instruction.clone()
13188    } else {
13189        format!(
13190            "{}\n\n# Explicit source inputs already prepared\n\n{}",
13191            state.instruction,
13192            inputs.join("\n")
13193        )
13194    }
13195}
13196
13197/// Does the winner have no commits ahead of the base it would land on?
13198/// Any failure to find out reads as "not empty": the merge then behaves as it
13199/// always did rather than refusing on a guess.
13200async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13201    let base = &state.base_branch;
13202    let mut against = base.clone();
13203    if mode == MergeMode::Pr {
13204        // A pull request lands on the remote's base, never the local branch
13205        // of the same name: if that cannot be read, "not empty" is the safe
13206        // answer.
13207        let remote = &state.config.merge.remote;
13208        let tracking = format!("{remote}/{base}");
13209        let fetched = git::fetch(repo, remote, base).await;
13210        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13211            against = tracking;
13212        } else {
13213            return false;
13214        }
13215    }
13216    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13217}
13218
13219/// Why nothing was opened for an empty winner, with what the task's own
13220/// references resolved to.
13221fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13222    let mut detail = format!(
13223        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13224         nothing to open a pull request for"
13225    );
13226    detail.push_str(
13227        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13228    );
13229    detail
13230}
13231
13232/// What the `Pr` merge does once it knows whether the branch already has an
13233/// open pull request.
13234#[derive(Debug, PartialEq, Eq)]
13235enum PrPlan {
13236    Create,
13237    Adopt { url: String, title: String },
13238    Stop(String),
13239}
13240
13241/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
13242/// a failed lookup -> stop with the real reason. Never guesses.
13243fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13244    match found {
13245        Ok(land::OpenPr::None) => PrPlan::Create,
13246        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13247        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13248            "several open pull requests exist for this branch, not picking one: {}",
13249            urls.join(" ")
13250        )),
13251        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13252    }
13253}
13254
13255/// `gh pr create`, returning the PR url.
13256pub(crate) async fn gh_pr_create(
13257    cwd: &Path,
13258    base: &str,
13259    head: &str,
13260    title: &str,
13261    body: &str,
13262) -> Result<String> {
13263    let out = tokio::process::Command::new("gh")
13264        .args([
13265            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13266        ])
13267        .current_dir(cwd)
13268        .quiet()
13269        .stdin(std::process::Stdio::null())
13270        .output()
13271        .await
13272        .context("spawn gh")?;
13273    if out.status.success() {
13274        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13275    } else {
13276        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13277    }
13278}
13279
13280/// Never force deletion of uncommitted work, including unknown legacy trees.
13281async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13282    match std::fs::symlink_metadata(path) {
13283        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13284            // A previous cleanup may have removed the tree before a branch
13285            // deletion or state save failed. Unregister it if still present.
13286            git::worktree_remove_clean(repo, path).await?;
13287            return Ok(());
13288        }
13289        Err(e) => return Err(e.into()),
13290        Ok(_) => {}
13291    }
13292    if !git::is_clean(path).await? {
13293        bail!(
13294            "worktree has uncommitted work; refusing removal: {}",
13295            path.display()
13296        );
13297    }
13298    if !git::worktree_remove_clean(repo, path).await? {
13299        bail!("worktree removal refused: {}", path.display());
13300    }
13301    Ok(())
13302}
13303
13304/// Tear a run's worktrees and branches down.
13305///
13306/// `home` is where the updated `run.json` is saved (via
13307/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
13308/// a housekeeping pass already has its own honest `home` handed to it, and
13309/// falling through to the global here would write back through whichever
13310/// directory some other process or test pinned into that `OnceLock` first,
13311/// not the one the caller actually resolved its `runs` and `state` from.
13312pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13313    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13314        bail!("refusing fold: {reason}");
13315    }
13316    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13317        bail!("candidate preservation incomplete; refusing cleanup");
13318    }
13319    // Serialize ordinary folding with the same starts/resumes as target-only
13320    // eviction. Otherwise a second janitor could remove the whole worktree
13321    // around a live output borrower even though target eviction was fenced.
13322    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13323    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13324    let _driver = crate::driver::maintenance(home, &state.id)?;
13325    let mut output_guards = Vec::new();
13326    for entry in std::fs::read_dir(state.worktree_root())
13327        .into_iter()
13328        .flatten()
13329        .flatten()
13330    {
13331        if entry.path().join(".git").is_file() {
13332            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13333        }
13334    }
13335    let repo = state.repo.clone();
13336    // CLI, Web and automatic cleanup all enter here, independently of
13337    // Runner::fold_losers. Protect deletion and shared config release too.
13338    let wait = repository_wait(state, None);
13339    let _coordination = match coordinate_operation(state, &repo, &wait).await {
13340        Ok(lease) => lease,
13341        Err(e) => {
13342            state.save_under(home)?;
13343            return Err(e);
13344        }
13345    };
13346    // Confirmed merges dispose of their owned sources after durable evidence
13347    // capture. Preflight every source before removing anything.
13348    let _source_outputs = if state.status == RunStatus::Merged
13349        && state.merge.as_ref().is_some_and(|merge| merge.ok)
13350    {
13351        verify_merged_fold_sources(state, drop_winner, home).await?
13352    } else {
13353        Vec::new()
13354    };
13355    let root = state.worktree_root();
13356    let winner = state.tally.as_ref().map(|t| t.winner);
13357    let mut removed = Vec::new();
13358
13359    for i in 0..state.candidates.len() {
13360        let c = state.candidates[i].clone();
13361        let is_winner = Some(c.label) == winner;
13362        if is_winner && !drop_winner {
13363            continue;
13364        }
13365        let existed = c.worktree.exists();
13366        // A vanished directory can still have a Git registration that keeps
13367        // its branch checked out. Remove that registration before the branch.
13368        remove_preserved_worktree(&repo, &c.worktree).await?;
13369        if existed {
13370            removed.push(c.worktree.to_string_lossy().into_owned());
13371        }
13372        // A branch handed to a later run (and its pull request) is not this
13373        // run's to delete.
13374        let handed_over = state.released_branches.contains(&c.branch);
13375        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13376            if !git::branch_delete(&repo, &c.branch).await? {
13377                bail!("candidate branch removal refused");
13378            }
13379            removed.push(c.branch.clone());
13380        }
13381        state.candidates[i].folded = true;
13382    }
13383
13384    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13385        let path = name.path();
13386        let keep = !drop_winner
13387            && winner.is_some_and(|w| {
13388                path.file_name()
13389                    .is_some_and(|n| n == format!("cand-{w}").as_str())
13390            });
13391        if keep {
13392            continue;
13393        }
13394        remove_preserved_worktree(&repo, &path).await?;
13395        removed.push(path.to_string_lossy().into_owned());
13396    }
13397
13398    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
13399    // judge worktrees, so once the loop above has cleared all of them out,
13400    // the parent is a bare directory nobody else was ever going to remove -
13401    // git only ever managed what was inside it. Left alone, one of these
13402    // accumulates per fully-folded run; the operator's own machine had 74.
13403    // `remove_if_empty` re-checks rather than assuming: a run whose winner
13404    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
13405    // so does anything a run never claimed that happens to share the bay.
13406    remove_if_empty(&root);
13407
13408    if state.enabled_worktree_config && drop_winner {
13409        // A release, not a raw disable: some sibling run in this repository
13410        // may still hold its own reference (see `git::acquire_worktree_config`),
13411        // and only the last release actually turns the setting back off.
13412        git::release_worktree_config(&repo).await.ok();
13413        state.enabled_worktree_config = false;
13414    }
13415    state.save_under(home)?;
13416    Ok(removed)
13417}
13418
13419async fn verify_merged_fold_sources(
13420    state: &RunState,
13421    drop_winner: bool,
13422    home: &Path,
13423) -> Result<Vec<std::fs::File>> {
13424    let winner = state.tally.as_ref().map(|t| t.winner);
13425    let mut heads = BTreeSet::new();
13426    for candidate in &state.candidates {
13427        if Some(candidate.label) == winner && !drop_winner {
13428            continue;
13429        }
13430        anyhow::ensure!(
13431            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13432            "candidate recovery source is outside the inventoried run root: {}",
13433            candidate.worktree.display()
13434        );
13435        match std::fs::symlink_metadata(&candidate.worktree) {
13436            Ok(_) => {}
13437            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13438            Err(error) => return Err(error.into()),
13439        }
13440        if !state.released_branches.contains(&candidate.branch)
13441            && git::branch_exists(&state.repo, &candidate.branch).await?
13442        {
13443            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13444        }
13445    }
13446    let mut cargo_guards = Vec::new();
13447    let entries = match std::fs::read_dir(state.worktree_root()) {
13448        Ok(entries) => Some(entries),
13449        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13450        Err(error) => return Err(error.into()),
13451    };
13452    for entry in entries.into_iter().flatten() {
13453        let path = entry?.path();
13454        let metadata = std::fs::symlink_metadata(&path)?;
13455        anyhow::ensure!(
13456            metadata.is_dir() && !metadata.file_type().is_symlink(),
13457            "non-directory or symlink recovery source retained: {}",
13458            path.display()
13459        );
13460        if !drop_winner
13461            && winner.is_some_and(|w| {
13462                path.file_name()
13463                    .is_some_and(|name| name == format!("cand-{w}").as_str())
13464            })
13465        {
13466            continue;
13467        }
13468        anyhow::ensure!(
13469            path.join(".git").is_file(),
13470            "unregistered recovery source retained: {}",
13471            path.display()
13472        );
13473        anyhow::ensure!(
13474            git::git(
13475                &path,
13476                &[
13477                    "status",
13478                    "--porcelain",
13479                    "--untracked-files=all",
13480                    "--ignored",
13481                    "--",
13482                    ".",
13483                    ":(exclude)target"
13484                ]
13485            )
13486            .await?
13487            .is_empty(),
13488            "source changes or ignored recovery files retained: {}",
13489            path.display()
13490        );
13491        match std::fs::symlink_metadata(path.join("target")) {
13492            Ok(_) => {
13493                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13494            }
13495            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13496            Err(error) => return Err(error.into()),
13497        }
13498        for checkpoint in [
13499            "rebase-merge",
13500            "rebase-apply",
13501            "sequencer",
13502            "CHERRY_PICK_HEAD",
13503            "MERGE_HEAD",
13504            "REBASE_HEAD",
13505            "REVERT_HEAD",
13506            "BISECT_LOG",
13507            "AUTO_MERGE",
13508            "MERGE_MSG",
13509        ] {
13510            let location = git::git(
13511                &path,
13512                &[
13513                    "rev-parse",
13514                    "--path-format=absolute",
13515                    "--git-path",
13516                    checkpoint,
13517                ],
13518            )
13519            .await?;
13520            anyhow::ensure!(
13521                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13522                if e.kind() == std::io::ErrorKind::NotFound),
13523                "Git recovery checkpoint retained: {checkpoint}"
13524            );
13525        }
13526        heads.insert(git::rev_parse(&path, "HEAD").await?);
13527    }
13528    crate::fold_evidence::preserve(state, home, &heads).await?;
13529
13530    Ok(cargo_guards)
13531}
13532
13533/// Remove `dir` if it exists and has nothing in it.
13534///
13535/// Best-effort and silent by design: a directory that is not empty (a run
13536/// whose winner is still parked there, a stray file some other process left)
13537/// is exactly the case this must refuse, and a directory that is already gone
13538/// is not a failure worth reporting either. `std::fs::remove_dir` itself
13539/// already refuses a non-empty directory, so the emptiness check below is
13540/// belt, not suspenders - it is what keeps this from ever attempting the
13541/// removal in the case that matters, rather than trusting `remove_dir`'s
13542/// error path to have no side effects if it ever changed.
13543fn remove_if_empty(dir: &Path) {
13544    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13545        std::fs::remove_dir(dir).ok();
13546    }
13547}
13548
13549/// Severity of the worst open finding in the last review round, for reporting.
13550pub fn worst_open(state: &RunState) -> Option<Severity> {
13551    state
13552        .reviews
13553        .last()?
13554        .reviews
13555        .iter()
13556        .flat_map(|r| r.findings.iter())
13557        .map(|f| f.severity)
13558        .max()
13559}
13560
13561/// `[worktree] setup` hides its products per worktree, which needs
13562/// `extensions.worktreeConfig`. Taken once per run and released with the run's
13563/// worktrees, like the hook's; every place that runs setup calls this first.
13564async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13565    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13566        git::acquire_worktree_config(repo).await?;
13567        state.enabled_worktree_config = true;
13568    }
13569    Ok(())
13570}
13571
13572#[cfg(test)]
13573mod tests {
13574    #[tokio::test]
13575    async fn launcher_base_fetch_requires_remote_success() {
13576        let tmp = tempfile::tempdir().unwrap();
13577        let repo = tmp.path().join("repo");
13578        let remote = tmp.path().join("remote");
13579        std::fs::create_dir(&repo).unwrap();
13580        std::fs::create_dir(&remote).unwrap();
13581        git::git(&remote, &["init", "--bare", "-b", "main"])
13582            .await
13583            .unwrap();
13584        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13585        git::git(
13586            &repo,
13587            &[
13588                "-c",
13589                "user.name=Fixture",
13590                "-c",
13591                "user.email=fixture@example.invalid",
13592                "commit",
13593                "--allow-empty",
13594                "-m",
13595                "fixture",
13596            ],
13597        )
13598        .await
13599        .unwrap();
13600        git::git(
13601            &repo,
13602            &["remote", "add", "origin", remote.to_str().unwrap()],
13603        )
13604        .await
13605        .unwrap();
13606        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13607        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13608        assert_eq!(
13609            resolve_base(&repo, "main", "origin", None)
13610                .await
13611                .unwrap()
13612                .commit,
13613            head
13614        );
13615        #[cfg(unix)]
13616        {
13617            let policy: crate::config::Coordination = toml::from_str(
13618                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13619            ).unwrap();
13620            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13621            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13622            let held = crate::repo_lock::SeatLease::operation_with(
13623                &repo,
13624                Some(&policy),
13625                &crate::repo_lock::WaitPolicy::default(),
13626            )
13627            .await
13628            .unwrap();
13629            assert!(held.is_some());
13630            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13631            tokio::pin!(fetch);
13632            assert!(
13633                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13634                    .await
13635                    .is_err()
13636            );
13637            assert!(
13638                !repo.join(".git/FETCH_HEAD").exists(),
13639                "no fetch before admission"
13640            );
13641            drop(held);
13642            let fetched = fetch.await.unwrap();
13643            assert_eq!(fetched.commit, head);
13644            assert_eq!(fetched.remote, "origin");
13645            assert_eq!(fetched.branch, "main");
13646        }
13647        // A surviving tracking ref is not successful fetch evidence.
13648        git::git(
13649            &repo,
13650            &[
13651                "remote",
13652                "set-url",
13653                "origin",
13654                tmp.path().join("missing").to_str().unwrap(),
13655            ],
13656        )
13657        .await
13658        .unwrap();
13659        let error = resolve_base(&repo, "main", "origin", None)
13660            .await
13661            .unwrap_err()
13662            .to_string();
13663        assert!(error.contains("cannot read origin/main"));
13664        assert!(error.contains("refusing to branch off"));
13665    }
13666
13667    #[test]
13668    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13669        let leaked = vec!["chore(deps): update a crate".to_owned()];
13670        let own = "fix(daemon): apply a chosen action";
13671        assert!(should_retitle("", own, &leaked));
13672        assert!(should_retitle(
13673            &format!("{REVIEW_PROMPT_OPENING} `x`"),
13674            own,
13675            &leaked
13676        ));
13677        assert!(should_retitle(
13678            "chore: land candidate A of run 1",
13679            own,
13680            &leaked
13681        ));
13682        assert!(should_retitle(
13683            "magi: candidate A (uncommitted work)",
13684            own,
13685            &leaked
13686        ));
13687        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13688        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13689        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13690    }
13691
13692    #[test]
13693    fn pr_merge_plan_creates_adopts_or_stops() {
13694        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13695        assert_eq!(
13696            pr_merge_plan(Ok(land::OpenPr::One {
13697                url: "u".into(),
13698                title: "t".into()
13699            })),
13700            PrPlan::Adopt {
13701                url: "u".into(),
13702                title: "t".into()
13703            }
13704        );
13705        let PrPlan::Stop(many) =
13706            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13707        else {
13708            panic!("many must stop");
13709        };
13710        assert!(many.contains('a') && many.contains('b'));
13711        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13712            panic!("a failed lookup must stop");
13713        };
13714        assert!(err.contains("bad token"));
13715    }
13716
13717    use super::*;
13718    use crate::run::GateStatus;
13719    use std::collections::BTreeMap;
13720    use std::time::Duration;
13721
13722    fn conductor() -> AgentSpec {
13723        AgentSpec {
13724            id: "conductor".to_owned(),
13725            kind: crate::config::AgentKind::Command,
13726            model: None,
13727            command: vec!["true".to_owned()],
13728            extra_args: Vec::new(),
13729            env: BTreeMap::new(),
13730            prompt_delivery: None,
13731            max_turn_secs: None,
13732        }
13733    }
13734
13735    fn spec(id: &str) -> AgentSpec {
13736        AgentSpec {
13737            id: id.to_owned(),
13738            kind: crate::config::AgentKind::Command,
13739            model: None,
13740            command: vec!["true".to_owned()],
13741            extra_args: Vec::new(),
13742            env: BTreeMap::new(),
13743            prompt_delivery: None,
13744            max_turn_secs: None,
13745        }
13746    }
13747
13748    fn ids(xs: &[&str]) -> BTreeSet<String> {
13749        xs.iter().map(|s| (*s).to_owned()).collect()
13750    }
13751
13752    #[test]
13753    fn pick_successor_skips_an_agent_another_seat_holds() {
13754        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
13755        let roster = [spec("a"), spec("b"), spec("c")];
13756        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13757        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13758    }
13759
13760    #[test]
13761    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13762        // The other seat started on c but was handed to d; c is free again.
13763        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13764        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13765        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13766        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13767        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13768    }
13769
13770    #[test]
13771    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13772        let roster = [spec("a"), spec("b")];
13773        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13774        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13775        let carried = ids(&["b"]);
13776        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13777        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13778    }
13779
13780    #[test]
13781    fn pick_successor_returns_none_without_any_untried_successor() {
13782        let roster = [spec("a"), spec("b")];
13783        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13784        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13785        let carried = ids(&["a"]);
13786        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13787    }
13788
13789    #[test]
13790    fn pick_successor_rescue_avoids_another_seats_occupant() {
13791        // b is a carried failure and free; a is held by the other seat.
13792        let roster = [spec("a"), spec("b"), spec("c")];
13793        let carried = ids(&["b", "c"]);
13794        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13795        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13796    }
13797
13798    #[test]
13799    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13800        let roster = [spec("a"), spec("b"), spec("c")];
13801        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13802        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13803    }
13804
13805    #[test]
13806    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13807        let roster = [spec("a"), spec("b"), spec("c")];
13808        let failed = ids(&["a", "b", "c"]);
13809        // Rescue looks at the whole roster, once per id, then runs out.
13810        let mut tried = ids(&["b"]);
13811        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13812        assert_eq!(first.id, "a");
13813        tried.insert(first.id.clone());
13814        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13815        assert_eq!(second.id, "c");
13816        tried.insert(second.id.clone());
13817        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13818    }
13819
13820    #[test]
13821    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13822        let roster = [spec("a"), spec("b")];
13823        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13824        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13825    }
13826
13827    #[test]
13828    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13829        let roster = [spec("a"), spec("b"), spec("c")];
13830        let h = SeatHistory {
13831            failed: ids(&["a"]),
13832            last_ok: Some("b".to_owned()),
13833            last_fail: None,
13834        };
13835        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13836        // A last answerer that left the roster, or later failed, is ignored.
13837        let gone = SeatHistory {
13838            last_ok: Some("zzz".to_owned()),
13839            ..h.clone()
13840        };
13841        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13842        let failed = SeatHistory {
13843            failed: ids(&["a", "b"]),
13844            last_ok: Some("b".to_owned()),
13845            last_fail: None,
13846        };
13847        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13848    }
13849
13850    #[test]
13851    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13852        let first = SeatState::new("review-1", "alpha", 7);
13853        let next = handover_seat("review-1", "gamma", 7);
13854        assert_ne!(first.claude_session, next.claude_session);
13855    }
13856
13857    #[test]
13858    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13859        let mut state = state_with_summary("x", "y");
13860        let a = handover_seat("review-1", "beta", state.next_seat_seed());
13861        let b = handover_seat("review-1", "beta", state.next_seat_seed());
13862        assert_ne!(a.claude_session, b.claude_session);
13863    }
13864
13865    #[test]
13866    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13867        let roster = [spec("a"), spec("b")];
13868        let h = SeatHistory {
13869            failed: ids(&["a", "b"]),
13870            ..SeatHistory::default()
13871        };
13872        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13873        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13874        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13875    }
13876
13877    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
13878    // fallback loop depends on to terminate: it must walk forward from the
13879    // seat's own position, never restart at the front of the roster, and it
13880    // must never hand back an id already tried, however many times that id
13881    // happens to appear.
13882
13883    #[test]
13884    fn failure_signature_ignores_numbers_and_paths() {
13885        assert_eq!(
13886            failure_signature("exited with Some(2) and no usable output"),
13887            failure_signature("exited with Some(137) and no usable output")
13888        );
13889        assert_eq!(
13890            failure_signature("process exited with no usable output (exit code 2)"),
13891            failure_signature("process exited with no usable output (exit code 137)")
13892        );
13893        for (old, new) in [
13894            (
13895                "exited with Some(2) and no usable output",
13896                "process exited with no usable output (exit code 2)",
13897            ),
13898            (
13899                "exited with None and no usable output",
13900                "process exited with no usable output (exit code unavailable)",
13901            ),
13902        ] {
13903            let previous = FailClass::Other(failure_signature(old));
13904            let current = FailClass::Other(failure_signature(new));
13905            assert_eq!(previous, current, "resumption retains the failure class");
13906            assert!(!should_hand_over(Some(&previous), &current));
13907        }
13908        assert_eq!(
13909            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13910            failure_signature("cannot open /var/x.txt: denied")
13911        );
13912        assert_ne!(failure_signature("boom"), failure_signature("bang"));
13913    }
13914
13915    #[test]
13916    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13917        let other = FailClass::Other("x".into());
13918        assert!(should_hand_over(None, &FailClass::Quota));
13919        assert!(should_hand_over(Some(&other), &FailClass::Quota));
13920        assert!(should_hand_over(
13921            Some(&FailClass::Timeout),
13922            &FailClass::Timeout
13923        ));
13924        assert!(should_hand_over(None, &other));
13925        assert!(!should_hand_over(Some(&other), &other));
13926        assert!(should_hand_over(
13927            Some(&other),
13928            &FailClass::Other("y".into())
13929        ));
13930        // A quota or timeout in between ends the run of identical failures.
13931        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13932        assert!(should_hand_over(Some(&FailClass::Quota), &other));
13933    }
13934
13935    #[test]
13936    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13937        let a = SeatState::new("judge-1", "alpha", 7);
13938        let b = handover_seat("judge-1", "beta", 7);
13939        assert_ne!(a.claude_session, b.claude_session);
13940        assert_eq!(b.turns, 0);
13941    }
13942
13943    #[test]
13944    fn a_timeout_is_classified_apart_from_other_failures() {
13945        assert_eq!(
13946            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13947            Some(FailClass::Timeout)
13948        );
13949        assert!(matches!(
13950            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13951            Some(FailClass::Other(_))
13952        ));
13953    }
13954
13955    #[test]
13956    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13957        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13958        let tried = BTreeSet::from(["beta".to_owned()]);
13959        // beta sits at index 1; the next candidate is gamma, never alpha —
13960        // which is very likely a different candidate slot's own agent.
13961        let next = next_untried_in_roster(&roster, 1, &tried);
13962        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13963    }
13964
13965    #[test]
13966    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13967        let roster = vec![spec("alpha"), spec("beta")];
13968        let tried = BTreeSet::from(["beta".to_owned()]);
13969        // beta is the roster's last entry: nothing follows it, and alpha —
13970        // earlier in the roster, almost certainly a different candidate
13971        // slot's own agent — must not be reached by wrapping back to it.
13972        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13973    }
13974
13975    #[test]
13976    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13977        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13978        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13979        // beta (index 1) and gamma (index 2, the only entry after it) have
13980        // both been tried; alpha (index 0) never has, but it comes before
13981        // beta's own position, so there is nothing further for this seat.
13982        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13983    }
13984
13985    #[test]
13986    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13987        let roster = vec![spec("a"), spec("a"), spec("b")];
13988        let tried = BTreeSet::from(["a".to_owned()]);
13989        let next = next_untried_in_roster(&roster, 0, &tried);
13990        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13991    }
13992
13993    #[test]
13994    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13995        let roster = vec![spec("a"), spec("b")];
13996        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13997        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13998    }
13999
14000    #[test]
14001    fn remove_if_empty_only_ever_takes_a_bare_directory() {
14002        let dir = tempfile::tempdir().unwrap();
14003        let bay = dir.path().join("ffff");
14004
14005        // Not there yet: nothing to do, nothing to panic on.
14006        remove_if_empty(&bay);
14007        assert!(!bay.exists());
14008
14009        // Something still inside - the winner's worktree, or a stray file -
14010        // keeps the directory standing.
14011        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14012        remove_if_empty(&bay);
14013        assert!(bay.exists(), "non-empty directory must survive");
14014
14015        // Once the last entry is gone, so is the directory itself.
14016        std::fs::remove_dir(bay.join("cand-A")).unwrap();
14017        remove_if_empty(&bay);
14018        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14019    }
14020
14021    // `round_is_clean` is the exact decision this task fixed: a round with a
14022    // seat that never answered must not read the same as a round every seat
14023    // actually reviewed. These are deterministic and process-free by design —
14024    // the equivalent end-to-end check (a real reviewer timing out under a
14025    // live graph run) is a genuine race against wall-clock contention, and a
14026    // spawn slow enough to blow even a generous budget under a loaded test
14027    // run must not turn this specific regression check flaky.
14028
14029    #[test]
14030    fn a_full_panel_that_found_nothing_is_clean() {
14031        assert!(round_is_clean(
14032            0,
14033            true,
14034            2,
14035            2,
14036            0,
14037            IncompleteReviewPolicy::Block
14038        ));
14039    }
14040
14041    #[test]
14042    fn a_missing_seat_is_never_clean_under_the_default_policy() {
14043        assert!(!round_is_clean(
14044            0,
14045            true,
14046            1,
14047            2,
14048            0,
14049            IncompleteReviewPolicy::Block
14050        ));
14051    }
14052
14053    #[test]
14054    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14055        assert!(!round_is_clean(
14056            1,
14057            true,
14058            1,
14059            2,
14060            0,
14061            IncompleteReviewPolicy::Warn
14062        ));
14063    }
14064
14065    #[test]
14066    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14067        assert!(round_is_clean(
14068            0,
14069            true,
14070            1,
14071            2,
14072            0,
14073            IncompleteReviewPolicy::Warn
14074        ));
14075    }
14076
14077    #[test]
14078    fn a_full_panel_with_an_open_finding_is_not_clean() {
14079        assert!(!round_is_clean(
14080            1,
14081            true,
14082            2,
14083            2,
14084            0,
14085            IncompleteReviewPolicy::Block
14086        ));
14087    }
14088
14089    #[test]
14090    fn a_full_panel_with_a_red_e2e_is_not_clean() {
14091        assert!(!round_is_clean(
14092            0,
14093            false,
14094            2,
14095            2,
14096            0,
14097            IncompleteReviewPolicy::Block
14098        ));
14099    }
14100
14101    // The stall this task closes: under the default `block` policy, a seat
14102    // missing only because it was rate limited must not force a wait for a
14103    // session limit that will not lift by the next round. `round_is_clean`
14104    // is where that quorum carve-out lives; the review loop around it never
14105    // changes what a reviewer's vote or a finding's severity means.
14106
14107    #[test]
14108    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14109        // 1 of 2 answered, and the one missing was quota'd — the exact
14110        // "review-2 rate limited (quota)" shape from the field report.
14111        assert!(round_is_clean(
14112            0,
14113            true,
14114            1,
14115            2,
14116            1,
14117            IncompleteReviewPolicy::Block
14118        ));
14119    }
14120
14121    #[test]
14122    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14123        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
14124        // not a quota loss (`quota_missing` stays 0) — worth another try.
14125        assert!(!round_is_clean(
14126            0,
14127            true,
14128            1,
14129            2,
14130            0,
14131            IncompleteReviewPolicy::Block
14132        ));
14133    }
14134
14135    #[test]
14136    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14137        assert!(!round_is_clean(
14138            1,
14139            true,
14140            1,
14141            2,
14142            1,
14143            IncompleteReviewPolicy::Block
14144        ));
14145        assert!(!round_is_clean(
14146            0,
14147            false,
14148            1,
14149            2,
14150            1,
14151            IncompleteReviewPolicy::Block
14152        ));
14153    }
14154
14155    #[test]
14156    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14157        // Every seat quota'd, nobody answered: there is no panel to decide
14158        // on, so this must fall through to the existing block-and-retry
14159        // fallback rather than call an unreviewed patch clean.
14160        assert!(!round_is_clean(
14161            0,
14162            true,
14163            0,
14164            2,
14165            2,
14166            IncompleteReviewPolicy::Block
14167        ));
14168    }
14169
14170    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14171        CommandOutcome {
14172            command: "test".to_owned(),
14173            code,
14174            output_tail: String::new(),
14175            duration_ms: 0,
14176            resource_blocked,
14177        }
14178    }
14179
14180    #[cfg(unix)]
14181    #[tokio::test]
14182    async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14183        ask_test_home();
14184        let temp = tempfile::tempdir().unwrap();
14185        let mut runner = runner_at(RunStatus::Gating);
14186        let shell = vec!["sh".to_owned(), "-c".to_owned()];
14187        let commands = vec![
14188            "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14189            "touch must-not-run".to_owned(),
14190        ];
14191        let (outcomes, retried, pids) = run_e2e_with_retry(
14192            &mut runner.state,
14193            &shell,
14194            &commands,
14195            temp.path(),
14196            Duration::from_secs(5),
14197            "fixture",
14198        )
14199        .await;
14200        assert_eq!(outcomes.len(), 1);
14201        assert!(outcomes[0].resource_blocked);
14202        assert!(!outcomes[0].build_failed());
14203        assert!(!retried);
14204        assert!(pids.is_empty());
14205        assert!(!temp.path().join("must-not-run").exists());
14206        assert_eq!(
14207            runner.state.resource_block.as_ref().unwrap().kind,
14208            "incremental cache admission"
14209        );
14210    }
14211
14212    #[test]
14213    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14214        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14215        assert!(
14216            !verify_inconclusive(&[outcome(Some(1), false)]),
14217            "an ordinary failure is still evidence about the patch"
14218        );
14219        assert!(verify_inconclusive(&[outcome(None, true)]));
14220        assert!(
14221            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14222            "one inconclusive outcome taints the whole batch"
14223        );
14224        assert!(!verify_inconclusive(&[]));
14225    }
14226
14227    #[tokio::test]
14228    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14229        // Alive for the first two checks, then dead - confirms the loop
14230        // actually re-polls rather than deciding once and sleeping out the
14231        // ceiling regardless.
14232        let calls = std::sync::atomic::AtomicUsize::new(0);
14233        let started = Instant::now();
14234        wait_for_pids_with(
14235            &[123],
14236            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14237            Duration::from_millis(5),
14238            Duration::from_secs(5),
14239        )
14240        .await;
14241        assert!(
14242            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14243            "must keep checking rather than deciding on the first answer"
14244        );
14245        assert!(
14246            started.elapsed() < Duration::from_secs(1),
14247            "must return the moment it is confirmed dead, not wait out the ceiling"
14248        );
14249    }
14250
14251    #[tokio::test]
14252    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14253        let started = Instant::now();
14254        wait_for_pids_with(
14255            &[123],
14256            |_| true, // never reports dead
14257            Duration::from_millis(5),
14258            Duration::from_millis(30),
14259        )
14260        .await;
14261        let elapsed = started.elapsed();
14262        assert!(
14263            elapsed >= Duration::from_millis(30),
14264            "must not give up before its own ceiling: {elapsed:?}"
14265        );
14266        assert!(
14267            elapsed < Duration::from_secs(1),
14268            "must not wait past its own ceiling either: {elapsed:?}"
14269        );
14270    }
14271
14272    #[tokio::test]
14273    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14274        let started = Instant::now();
14275        wait_for_pids_with(
14276            &[],
14277            |_| true,
14278            Duration::from_secs(5),
14279            Duration::from_secs(5),
14280        )
14281        .await;
14282        assert!(
14283            started.elapsed() < Duration::from_millis(200),
14284            "an empty pid list has nothing to confirm"
14285        );
14286    }
14287
14288    // `review_conclusion` is the exact decision the review hand-off task
14289    // fixed: a round budget spent (or a tree that stopped moving) must not
14290    // collapse into `Blocked` regardless of what verification actually
14291    // said. Deterministic and process-free for the same reason the
14292    // `round_is_clean` family above is.
14293    fn review_round(
14294        clean: bool,
14295        blocking: usize,
14296        answered: usize,
14297        expected: usize,
14298        progressed: bool,
14299        e2e_ok: bool,
14300    ) -> ReviewRound {
14301        ReviewRound {
14302            round: 1,
14303            head: "h".to_owned(),
14304            verified_head: None,
14305            verified_at: None,
14306            reviews: Vec::new(),
14307            e2e: vec![CommandOutcome {
14308                command: "test".to_owned(),
14309                code: Some(if e2e_ok { 0 } else { 1 }),
14310                output_tail: String::new(),
14311                duration_ms: 0,
14312                resource_blocked: false,
14313            }],
14314            verify_retried: false,
14315            e2e_deferred: false,
14316            e2e_defer_reason: None,
14317            fix: None,
14318            blocking,
14319            answered,
14320            expected,
14321            clean,
14322            progressed,
14323            vote_split: false,
14324            reconsideration: Vec::new(),
14325            verdict: None,
14326        }
14327    }
14328
14329    #[test]
14330    fn review_conclusion_is_none_when_nothing_has_run() {
14331        assert_eq!(review_conclusion(&[], 3), None);
14332    }
14333
14334    #[test]
14335    fn review_conclusion_is_none_while_rounds_remain() {
14336        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14337        assert_eq!(review_conclusion(&rounds, 3), None);
14338    }
14339
14340    #[test]
14341    fn review_conclusion_is_gating_once_a_round_is_clean() {
14342        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14343        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14344    }
14345
14346    #[test]
14347    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14348        let rounds = vec![
14349            review_round(false, 1, 2, 2, true, true),
14350            review_round(false, 1, 2, 2, true, true),
14351        ];
14352        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14353    }
14354
14355    #[test]
14356    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14357        let rounds = vec![
14358            review_round(false, 1, 2, 2, true, true),
14359            review_round(false, 1, 2, 2, true, false),
14360        ];
14361        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14362    }
14363
14364    #[test]
14365    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14366        // Magi never got a command to run against this round's own head — a
14367        // resource-blocked attempt, not a red one — so this must never
14368        // settle on `Blocked` the way a genuine e2e failure would. `None`
14369        // here is what tells `Runner::review_loop` to retry the check
14370        // itself rather than trust this cheap recomputation with a verdict
14371        // it cannot actually produce.
14372        let mut blocked = review_round(false, 1, 2, 2, true, false);
14373        blocked.e2e[0].resource_blocked = true;
14374        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14375        assert_eq!(review_conclusion(&rounds, 2), None);
14376    }
14377
14378    #[test]
14379    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14380        // Missing input, not a verified tree — never a hand-off candidate.
14381        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14382        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14383    }
14384
14385    #[test]
14386    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14387        let rounds = vec![
14388            review_round(false, 1, 2, 2, false, true),
14389            review_round(false, 1, 2, 2, false, true),
14390        ];
14391        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14392    }
14393
14394    fn secs(n: u64) -> Duration {
14395        Duration::from_secs(n)
14396    }
14397
14398    /// A throwaway repo with one commit on `main`, for tests that need `merge`
14399    /// to make real (and, if it runs at all, real*ly fail*) git calls.
14400    #[tokio::test]
14401    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14402        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14403        if std::env::var_os(CHILD).is_none() {
14404            // Other tests reconcile every run in the process-global home,
14405            // briefly taking owner locks. This ancestry test expects an idle
14406            // predecessor, so give it its own process and pinned test home.
14407            let output = std::process::Command::new(std::env::current_exe().unwrap())
14408                .args([
14409                    "--exact",
14410                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14411                    "--nocapture",
14412                ])
14413                .env(CHILD, "1")
14414                .env_remove("MAGI_WORKER_HOME")
14415                .env_remove("MAGI_WORKER_ID")
14416                .output()
14417                .unwrap();
14418            assert!(
14419                output.status.success()
14420                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14421                "{}\n{}",
14422                String::from_utf8_lossy(&output.stdout),
14423                String::from_utf8_lossy(&output.stderr)
14424            );
14425            return;
14426        }
14427        ask_test_home();
14428        let tmp = tempfile::tempdir().unwrap();
14429        let repo = tmp.path().join("repo");
14430        std::fs::create_dir_all(&repo).unwrap();
14431        init_repo(&repo);
14432        let g = |args: &[&str]| {
14433            let out = std::process::Command::new("git")
14434                .args(args)
14435                .current_dir(&repo)
14436                .quiet()
14437                .output()
14438                .unwrap();
14439            assert!(
14440                out.status.success(),
14441                "{}",
14442                String::from_utf8_lossy(&out.stderr)
14443            );
14444        };
14445        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14446        g(&["checkout", "-b", "historical"]);
14447        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14448        g(&["add", "."]);
14449        g(&["commit", "-m", "old alternative"]);
14450        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14451        g(&["checkout", "-b", "magi/task-selected", &base]);
14452        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14453        g(&["add", "."]);
14454        g(&["commit", "-m", "measurement infrastructure"]);
14455        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14456        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14457        g(&["add", "."]);
14458        g(&["commit", "-m", "dependent implementation"]);
14459        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14460        g(&["checkout", "main"]);
14461        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14462        g(&["add", "."]);
14463        g(&["commit", "-m", "unrelated base change"]);
14464        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14465        let origin = tmp.path().join("origin.git");
14466        g(&[
14467            "clone",
14468            "--bare",
14469            repo.to_str().unwrap(),
14470            origin.to_str().unwrap(),
14471        ]);
14472        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14473        g(&["fetch", "origin"]);
14474        let selection = crate::pr_recovery::Selection {
14475            url: "https://example.test/project/pull/1".into(),
14476            branch: "magi/task-selected".into(),
14477            head: head.clone(),
14478            source_head: None,
14479            source_run: None,
14480            source_proof: None,
14481            disposition: Some(crate::pr_recovery::Disposition::Review),
14482            instruction: "preserve measured behavior".into(),
14483            approved_at: jiff::Timestamp::now(),
14484        };
14485        let mut template = runner_at(RunStatus::Reviewing);
14486        template.state.repo = repo.clone();
14487        template.state.base_commit = current.clone();
14488        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14489        template.state.origin = Some(crate::run::Origin::queue("selected"));
14490        template.state.pr_head = Some(selection.branch.clone());
14491        template.state.pr_recovery = Some(selection.clone());
14492        let mut runner =
14493            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14494                .await
14495                .unwrap();
14496        let wt = runner.state.winner().unwrap().worktree.clone();
14497        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14498        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14499        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14500        assert!(runner.state.seeds.is_empty());
14501        assert_eq!(runner.state.instruction, "preserve measured behavior");
14502        runner.sync_to_base().await.unwrap();
14503        assert_eq!(
14504            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14505            "prerequisite"
14506        );
14507        assert_eq!(
14508            std::fs::read_to_string(wt.join("implementation")).unwrap(),
14509            "measured implementation"
14510        );
14511        assert_eq!(
14512            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14513            "current main change"
14514        );
14515        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14516        assert_eq!(
14517            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14518            head
14519        );
14520
14521        // A failed selected-source setup stays inspectable, never resumable as a new competition.
14522        let mut partial = runner_at(RunStatus::Reviewing);
14523        partial.state.repo = repo.clone();
14524        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14525        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14526            run: Some("exit 1".into()),
14527            ..Default::default()
14528        }];
14529        let selection = runner.state.pr_recovery.clone().unwrap();
14530        partial.state.pr_recovery = Some(selection.clone());
14531        partial.state.pr_head = Some(selection.branch.clone());
14532        partial.state.origin = Some(crate::run::Origin::queue("selected"));
14533        let id = partial.state.id.clone();
14534        let error = match Runner::open_selected(
14535            &repo,
14536            partial.state,
14537            partial.roles,
14538            1,
14539            runner.state.base_commit.clone(),
14540            selection,
14541        )
14542        .await
14543        {
14544            Ok(_) => panic!("setup must fail"),
14545            Err(e) => e,
14546        };
14547        assert_eq!(
14548            error
14549                .downcast_ref::<crate::run::StartupFailure>()
14550                .unwrap()
14551                .run,
14552            id
14553        );
14554        let failed = RunState::load(&id).unwrap();
14555        assert_eq!(failed.status, RunStatus::Failed);
14556        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14557        assert!(Runner::resume(&id).is_err());
14558        runner.state.status = RunStatus::Ready;
14559        runner.state.driver_exited = true;
14560        runner.state.save().unwrap();
14561        let mut retry = runner_at(RunStatus::Reviewing);
14562        retry.state.repo = repo.clone();
14563        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14564        let mut reselected = runner.state.pr_recovery.clone().unwrap();
14565        reselected.approved_at = jiff::Timestamp::now();
14566        retry.state.origin = Some(crate::run::Origin::queue("selected"));
14567        retry.state.pr_head = Some(reselected.branch.clone());
14568        retry.state.pr_recovery = Some(reselected.clone());
14569        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14570            .await
14571            .unwrap();
14572        let restarted = Runner::open_selected(
14573            &repo,
14574            retry.state,
14575            retry.roles,
14576            1,
14577            runner.state.base_commit.clone(),
14578            reselected,
14579        )
14580        .await
14581        .unwrap();
14582        assert_eq!(
14583            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14584                .await
14585                .unwrap(),
14586            head
14587        );
14588        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14589        assert_eq!(
14590            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14591            head
14592        );
14593    }
14594
14595    #[tokio::test]
14596    #[cfg(unix)]
14597    async fn fallback_and_interrupted_restoration_require_coordination() {
14598        use std::os::fd::AsRawFd as _;
14599        ask_test_home();
14600        let tmp = tempfile::tempdir().unwrap();
14601        let mut runner = registered_runner(tmp.path()).await;
14602        runner.state.config.graph.timeout_resource_wait = 1;
14603        let winner = runner.state.candidates[0].clone();
14604        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14605        let leftover = winner.worktree.join("leftover");
14606        std::fs::write(&leftover, "untrusted edits").unwrap();
14607        let next = tmp.path().join("next-fixer");
14608        let mut failed = spec("first");
14609        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14610        let mut successor = spec("second");
14611        successor.command = vec![
14612            "sh".into(),
14613            "-c".into(),
14614            format!("touch '{}'; echo done", next.display()),
14615        ];
14616        let attempts = vec![
14617            (failed.clone(), "first".into()),
14618            (successor.clone(), "second".into()),
14619        ];
14620        let lock_path = runner.state.repo.join(".git/updater.lock");
14621        let external = std::fs::File::create(&lock_path).unwrap();
14622        assert_eq!(
14623            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14624            0
14625        );
14626        let artifacts = tmp.path().join("artifacts");
14627        let build = |spec, seat| SeatJob {
14628            spec,
14629            seat,
14630            cwd: winner.worktree.clone(),
14631            prompt: "fixture".into(),
14632            timeout: Duration::from_secs(10),
14633            allow_write: true,
14634            sessions: false,
14635            artifacts: artifacts.clone(),
14636            stem: "fix".into(),
14637            handover: None,
14638        };
14639        let (_, _, out) = runner
14640            .ask_fixer_among(
14641                attempts.clone(),
14642                "graft",
14643                None,
14644                Some((&winner.worktree, &head)),
14645                build,
14646            )
14647            .await;
14648        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14649        assert!(leftover.exists(), "contention must prevent clean");
14650        assert!(!next.exists());
14651        assert!(runner.state.handovers.is_empty());
14652        // Early interrupted-repair rollback uses the same protected route.
14653        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14654            from_run: "earlier".into(),
14655            generation: 1,
14656            mode: crate::recovery::Mode::VerifyRepair,
14657            reasons: Vec::new(),
14658            reviewed_head: None,
14659            rounds: Vec::new(),
14660            open_findings: Vec::new(),
14661            evidence: None,
14662            criteria_inherited: true,
14663            repair: crate::recovery::Repair::Started {
14664                before: head.clone(),
14665            },
14666        });
14667        assert!(
14668            runner
14669                .recovery_repair(&winner)
14670                .await
14671                .unwrap_err()
14672                .is::<crate::repo_lock::ResourceBlocked>()
14673        );
14674        assert!(matches!(
14675            runner.state.recovery.as_ref().unwrap().repair,
14676            crate::recovery::Repair::Started { .. }
14677        ));
14678        assert!(leftover.exists());
14679        drop(external);
14680        runner.recovery_repair(&winner).await.unwrap();
14681        assert!(!leftover.exists());
14682        assert!(matches!(
14683            runner.state.recovery.as_ref().unwrap().repair,
14684            crate::recovery::Repair::Done { .. }
14685        ));
14686        // A hard lock-open failure also leaves HEAD and untracked work intact.
14687        std::fs::remove_file(&lock_path).unwrap();
14688        std::fs::create_dir(&lock_path).unwrap();
14689        std::fs::write(&leftover, "untrusted edits").unwrap();
14690        let (_, _, out) = runner
14691            .ask_fixer_among(
14692                attempts,
14693                "graft",
14694                None,
14695                Some((&winner.worktree, &head)),
14696                build,
14697            )
14698            .await;
14699        assert!(matches!(out, AgentOutcome::Failed(_)));
14700        assert!(leftover.exists());
14701        assert!(!next.exists());
14702        assert_eq!(
14703            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14704            head
14705        );
14706    }
14707
14708    #[cfg(unix)]
14709    async fn registered_runner(root: &Path) -> Runner {
14710        repository_runner(root, crate::config::test_coordination()).await
14711    }
14712
14713    async fn repository_runner(
14714        root: &Path,
14715        coordination: Option<crate::config::Coordination>,
14716    ) -> Runner {
14717        ask_test_home();
14718        let repo = root.join("repo");
14719        std::fs::create_dir_all(&repo).unwrap();
14720        init_repo(&repo);
14721        if coordination.is_some() {
14722            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14723        }
14724        let mut runner = runner_at(RunStatus::Prep);
14725        runner.state.repo = repo.clone();
14726        runner.state.config.coordination = coordination;
14727        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14728        runner.state.config.graph.worktree_root = Some(root.join("wt"));
14729        runner.state.config.graph.advise = false;
14730        runner.roles.implementers = vec![spec("implement")];
14731        runner.roles.judges = vec![spec("judge")];
14732        runner.prep().await.unwrap();
14733        runner
14734    }
14735
14736    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14737
14738    #[tokio::test]
14739    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14740        let tmp = tempfile::tempdir().unwrap();
14741        let mut runner = repository_runner(tmp.path(), None).await;
14742        let prompts = runner.state.config.prompts.clone();
14743        let run = runner.state.id.clone();
14744        let ctx = WaveCtx {
14745            carry_seats: false,
14746            pauses: None,
14747            run: &run,
14748            node: "advise",
14749            prompts: &prompts,
14750            cache: None,
14751            round: None,
14752        };
14753        let jobs: Vec<_> = ["alpha", "beta"]
14754            .into_iter()
14755            .map(|id| SeatJob {
14756                spec: scripted(id, AUTH_OUTAGE),
14757                seat: SeatState::new(id, id, 7),
14758                prompt: "propose".into(),
14759                cwd: runner.state.candidates[0].worktree.clone(),
14760                timeout: Duration::from_secs(10),
14761                allow_write: false,
14762                sessions: false,
14763                artifacts: agent::artifacts_dir(&runner.state.dir()),
14764                stem: id.into(),
14765                handover: None,
14766            })
14767            .collect();
14768        let mut losses = Vec::new();
14769        let results = ask_wave_with(
14770            jobs.clone(),
14771            runner.sem.clone(),
14772            2,
14773            &[],
14774            &ctx,
14775            &mut losses,
14776            &mut runner.state,
14777            &|_: &str| Ok(()),
14778        )
14779        .await;
14780        assert!(
14781            results
14782                .iter()
14783                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14784        );
14785        assert_eq!(runner.state.quota.len(), 2);
14786        assert!(
14787            runner
14788                .state
14789                .quota
14790                .iter()
14791                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14792        );
14793        assert!(!runner.state.advise_attempted);
14794        runner.stop_for_resource().unwrap();
14795        assert_eq!(runner.state.status, RunStatus::Blocked);
14796        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14797        assert_eq!(
14798            runner.state.resource_block.as_ref().unwrap().kind,
14799            "agent authentication"
14800        );
14801        // A machine loss followed by an ordinary failure is not evidence
14802        // that the whole node was unavailable, and must not earn a refund.
14803        runner.state.resource_block = None;
14804        runner.state.quota.clear();
14805        let mut mixed = jobs;
14806        mixed[1].spec.command = vec!["false".into()];
14807        let results = ask_wave_with(
14808            mixed,
14809            runner.sem.clone(),
14810            0,
14811            &[],
14812            &ctx,
14813            &mut losses,
14814            &mut runner.state,
14815            &|_: &str| Ok(()),
14816        )
14817        .await;
14818        assert!(results.iter().all(|(_, res, _)| res.is_err()));
14819        assert!(runner.state.resource_block.is_none());
14820        assert_eq!(losses.len(), 1);
14821    }
14822
14823    #[tokio::test]
14824    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14825        let tmp = tempfile::tempdir().unwrap();
14826        let mut runner = repository_runner(tmp.path(), None).await;
14827        let implementer = scripted("implement", AUTH_OUTAGE);
14828        runner.roles.implementers = vec![implementer.clone()];
14829        runner.state.config.agents = vec![implementer];
14830        runner.implement().await.unwrap();
14831        assert_eq!(runner.state.status, RunStatus::Blocked);
14832        assert_eq!(runner.state.quota.len(), 1);
14833        assert_eq!(
14834            runner.state.quota[0].cause,
14835            agent::QuotaCause::Authentication
14836        );
14837        let candidate = &runner.state.candidates[0];
14838        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14839        assert_eq!(runner.state.seats["impl-A"].turns, 1);
14840    }
14841
14842    #[tokio::test]
14843    #[cfg(unix)]
14844    async fn parked_implementer_persists_completed_work() {
14845        for registered in [false, true] {
14846            let tmp = tempfile::tempdir().unwrap();
14847            let mut runner = registered_runner(tmp.path()).await;
14848            if !registered {
14849                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14850            }
14851            let marker = tmp.path().join("started");
14852            let release = tmp.path().join("release");
14853            let mut implementer = spec("implement");
14854            implementer.command = vec![
14855                "sh".into(),
14856                "-c".into(),
14857                format!(
14858                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14859                    marker.display(),
14860                    release.display()
14861                ),
14862            ];
14863            runner.roles.implementers = vec![implementer.clone()];
14864            runner.state.config.agents = vec![implementer];
14865            let pause = runner.pause.clone();
14866            let (result, ()) = tokio::join!(runner.implement(), async {
14867                tokio::time::timeout(Duration::from_secs(10), async {
14868                    while !marker.exists() {
14869                        tokio::time::sleep(Duration::from_millis(10)).await;
14870                    }
14871                })
14872                .await
14873                .unwrap();
14874                pause.park();
14875                std::fs::write(&release, "go").unwrap();
14876            });
14877            result.unwrap();
14878            let candidate = &runner.state.candidates[0];
14879            assert!(candidate.commits > 0, "{candidate:?}");
14880            assert!(candidate.summary.contains("saved-result"));
14881            assert_eq!(candidate.files, 1);
14882            let before = candidate.commits;
14883            runner.implement().await.unwrap();
14884            assert_eq!(runner.state.candidates[0].commits, before);
14885        }
14886    }
14887
14888    #[tokio::test]
14889    #[cfg(unix)]
14890    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14891        let tmp = tempfile::tempdir().unwrap();
14892        let mut runner = registered_runner(tmp.path()).await;
14893        let winner = runner.state.candidates[0].clone();
14894        let cache = tmp.path().join("cache");
14895        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14896        runner.state.config.graph.timeout_resource_wait = 0;
14897        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14898        let crate::cache::AcquireOutcome::Acquired(held) =
14899            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14900        else {
14901            panic!("cache must be free")
14902        };
14903        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14904            from_run: "earlier".into(),
14905            generation: 1,
14906            mode: crate::recovery::Mode::VerifyRepair,
14907            reasons: Vec::new(),
14908            reviewed_head: None,
14909            rounds: Vec::new(),
14910            open_findings: Vec::new(),
14911            evidence: None,
14912            criteria_inherited: true,
14913            repair: crate::recovery::Repair::NotStarted,
14914        });
14915        runner.recovery_repair(&winner).await.unwrap();
14916        assert!(matches!(
14917            runner.state.recovery.as_ref().unwrap().repair,
14918            crate::recovery::Repair::WaitingForRestore { .. }
14919        ));
14920        drop(held);
14921        runner.state.resource_block = None;
14922        runner.state.config.graph.timeout_resource_wait = 1;
14923        runner.recovery_repair(&winner).await.unwrap();
14924        assert!(matches!(
14925            runner.state.recovery.as_ref().unwrap().repair,
14926            crate::recovery::Repair::Done { .. }
14927        ));
14928        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14929    }
14930
14931    #[tokio::test]
14932    #[cfg(unix)]
14933    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14934        use std::os::fd::AsRawFd as _;
14935        for graft in [false, true] {
14936            let tmp = tempfile::tempdir().unwrap();
14937            let mut runner = registered_runner(tmp.path()).await;
14938            runner.state.config.graph.timeout_resource_wait = 1;
14939            let winner = runner.state.candidates[0].clone();
14940            let marker = tmp.path().join("fixer-started");
14941            let mut fixer = spec("fixer");
14942            fixer.command = vec![
14943                "sh".into(),
14944                "-c".into(),
14945                format!("echo started >> '{}'; echo done", marker.display()),
14946            ];
14947            runner.roles.fixer = Some(vec![fixer]);
14948            if graft {
14949                let mut loser = winner.clone();
14950                loser.label = 'B';
14951                loser.index = 1;
14952                runner.state.candidates.push(loser);
14953                runner.state.config.graph.graft_runner_up = true;
14954                runner.state.tally = Some(Tally {
14955                    first_choice: BTreeMap::new(),
14956                    borda: BTreeMap::new(),
14957                    winner: winner.label,
14958                    rankings: 1,
14959                    unanimous_initial: true,
14960                    deliberated: false,
14961                    changed_votes: 0,
14962                    unanimous_final: true,
14963                    tie_break: None,
14964                    judges: 1,
14965                    present: 1,
14966                    quorum: 1,
14967                    met_quorum: true,
14968                    uncontested: None,
14969                });
14970                runner.state.judgements.push(crate::run::Judgement {
14971                    judge: 1,
14972                    seat: "judge-1".into(),
14973                    agent: "judge".into(),
14974                    ranking: vec![winner.label, 'B'],
14975                    reasons: BTreeMap::new(),
14976                    confidence: None,
14977                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14978                    order: vec![0, 1],
14979                    failed: None,
14980                    duration_ms: 0,
14981                });
14982            } else {
14983                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14984                    from_run: "earlier".into(),
14985                    generation: 1,
14986                    mode: crate::recovery::Mode::VerifyRepair,
14987                    reasons: Vec::new(),
14988                    reviewed_head: None,
14989                    rounds: Vec::new(),
14990                    open_findings: Vec::new(),
14991                    evidence: None,
14992                    criteria_inherited: true,
14993                    repair: crate::recovery::Repair::NotStarted,
14994                });
14995            }
14996            let reader = crate::repo_lock::SeatLease::for_checkout(
14997                &winner.worktree,
14998                false,
14999                crate::config::test_coordination().as_ref(),
15000            )
15001            .await
15002            .unwrap();
15003            let pause = runner.pause.clone();
15004            let (result, ()) = tokio::join!(
15005                async {
15006                    if graft {
15007                        runner.graft().await
15008                    } else {
15009                        runner.recovery_repair(&winner).await
15010                    }
15011                },
15012                async {
15013                    tokio::time::sleep(Duration::from_millis(100)).await;
15014                    pause.park();
15015                }
15016            );
15017            result.unwrap();
15018            assert!(!marker.exists(), "a blocked seat must not spawn");
15019            // Exercise the persisted state, as a binary replacement/resume does.
15020            runner.state = RunState::load(&runner.state.id).unwrap();
15021            if graft {
15022                assert_eq!(
15023                    runner.state.graft.as_ref().unwrap().outcome,
15024                    GraftOutcome::WaitingForRestore
15025                );
15026            } else {
15027                assert!(matches!(
15028                    runner.state.recovery.as_ref().unwrap().repair,
15029                    crate::recovery::Repair::WaitingForRestore { .. }
15030                ));
15031            }
15032            drop(reader);
15033            runner.pause = Pause::new();
15034            runner.state.resource_block = None;
15035            // Deferred restoration must still respect an external lock owner.
15036            let external =
15037                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15038            assert_eq!(
15039                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15040                0
15041            );
15042            let leftover = winner.worktree.join("leftover");
15043            std::fs::write(&leftover, "earlier fallback edits").unwrap();
15044            let result = if graft {
15045                runner.graft().await
15046            } else {
15047                runner.recovery_repair(&winner).await
15048            };
15049            assert!(
15050                result
15051                    .unwrap_err()
15052                    .is::<crate::repo_lock::ResourceBlocked>()
15053            );
15054            assert!(leftover.exists());
15055            assert!(!marker.exists());
15056            drop(external);
15057            runner.state.resource_block = None;
15058            let result = if graft {
15059                runner.graft().await
15060            } else {
15061                runner.recovery_repair(&winner).await
15062            };
15063            result.unwrap();
15064            assert!(!leftover.exists(), "restore must precede retry");
15065            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15066            if graft {
15067                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15068            } else {
15069                assert!(matches!(
15070                    runner.state.recovery.as_ref().unwrap().repair,
15071                    crate::recovery::Repair::Done { .. }
15072                ));
15073            }
15074        }
15075    }
15076
15077    #[tokio::test]
15078    #[cfg(unix)]
15079    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15080        use std::os::fd::AsRawFd as _;
15081        let tmp = tempfile::tempdir().unwrap();
15082        let mut runner = registered_runner(tmp.path()).await;
15083        runner.state.config.graph.timeout_resource_wait = 1;
15084        let winner = runner.state.candidates[0].clone();
15085        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15086        runner.state.tally = Some(Tally {
15087            first_choice: BTreeMap::new(),
15088            borda: BTreeMap::new(),
15089            winner: winner.label,
15090            rankings: 0,
15091            unanimous_initial: false,
15092            deliberated: false,
15093            changed_votes: 0,
15094            unanimous_final: false,
15095            tie_break: None,
15096            judges: 0,
15097            present: 0,
15098            quorum: 0,
15099            met_quorum: true,
15100            uncontested: Some("fixture".into()),
15101        });
15102        runner.state.graft = Some(GraftRecord {
15103            pre: pre.clone(),
15104            post: None,
15105            lines: None,
15106            limit: None,
15107            agent: None,
15108            outcome: GraftOutcome::Running,
15109            requested: Vec::new(),
15110            adoption: None,
15111            adoption_error: None,
15112        });
15113        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15114        git::commit_all(&winner.worktree, "unfinished graft")
15115            .await
15116            .unwrap();
15117        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15118        let leftover = winner.worktree.join("leftover");
15119        std::fs::write(&leftover, "untrusted edits").unwrap();
15120        let path = runner.state.repo.join(".git/updater.lock");
15121        let external = std::fs::File::create(&path).unwrap();
15122        assert_eq!(
15123            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15124            0
15125        );
15126        assert!(
15127            runner
15128                .graft()
15129                .await
15130                .unwrap_err()
15131                .is::<crate::repo_lock::ResourceBlocked>()
15132        );
15133        assert!(matches!(
15134            runner.state.graft.as_ref().unwrap().outcome,
15135            GraftOutcome::Running
15136        ));
15137        assert_eq!(
15138            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15139            unfinished
15140        );
15141        assert!(leftover.exists());
15142        drop(external);
15143        std::fs::remove_file(&path).unwrap();
15144        std::fs::create_dir(&path).unwrap();
15145        assert!(runner.graft().await.is_err());
15146        assert!(matches!(
15147            runner.state.graft.as_ref().unwrap().outcome,
15148            GraftOutcome::Running
15149        ));
15150        assert_eq!(
15151            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15152            unfinished
15153        );
15154        assert!(leftover.exists());
15155        std::fs::remove_dir(&path).unwrap();
15156        runner.graft().await.unwrap();
15157        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15158        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15159        assert!(!leftover.exists());
15160    }
15161
15162    #[tokio::test]
15163    #[cfg(unix)]
15164    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15165        use std::os::fd::AsRawFd as _;
15166        let tmp = tempfile::tempdir().unwrap();
15167        let mut runner = registered_runner(tmp.path()).await;
15168        runner.state.prep_complete = Some(false);
15169        runner.state.config.graph.timeout_resource_wait = 1;
15170        runner.state.save().unwrap();
15171        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15172        assert_eq!(
15173            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15174            0
15175        );
15176        runner.execute().await.unwrap();
15177        let disk = RunState::load(&runner.state.id).unwrap();
15178        assert!(disk.driver_exited);
15179        assert_eq!(disk.driver_pid, None);
15180        assert_eq!(runner.state.status, RunStatus::Blocked);
15181        assert_eq!(
15182            runner.state.resource_block.as_ref().unwrap().kind,
15183            "repository coordination"
15184        );
15185        assert!(runner.state.prep_failure.is_none());
15186        assert!(runner.state.seats.is_empty());
15187        runner.pause.park();
15188        // Include headroom for Git subprocesses under parallel test load;
15189        // the repository contention deadline itself remains one second.
15190        tokio::time::timeout(Duration::from_secs(10), runner.execute())
15191            .await
15192            .unwrap()
15193            .unwrap();
15194        assert!(runner.state.parked);
15195        assert!(runner.state.prep_failure.is_none());
15196        let disk = RunState::load(&runner.state.id).unwrap();
15197        assert!(disk.driver_exited);
15198        assert_eq!(disk.driver_pid, None);
15199    }
15200
15201    #[tokio::test]
15202    #[cfg(unix)]
15203    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15204        use std::os::fd::AsRawFd as _;
15205        let tmp = tempfile::tempdir().unwrap();
15206        let mut runner = registered_runner(tmp.path()).await;
15207        runner.state.prep_complete = Some(false);
15208        runner.state.config.graph.timeout_resource_wait = 30;
15209        runner.state.save().unwrap();
15210        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15211        assert_eq!(
15212            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15213            0
15214        );
15215        assert!(
15216            tokio::time::timeout(Duration::from_millis(100), runner.execute())
15217                .await
15218                .is_err()
15219        );
15220        let disk = RunState::load(&runner.state.id).unwrap();
15221        assert!(disk.driver_exited);
15222        assert_eq!(disk.driver_pid, None);
15223        assert!(disk.status.resumable());
15224        assert!(runner.state.driver_exited);
15225        assert_eq!(runner.state.driver_pid, None);
15226        runner.state.save().unwrap();
15227        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15228    }
15229
15230    #[tokio::test]
15231    #[cfg(unix)]
15232    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15233        let tmp = tempfile::tempdir().unwrap();
15234        let mut runner = registered_runner(tmp.path()).await;
15235        let winner = runner.state.candidates[0].clone();
15236        let cache = tmp.path().join("cache");
15237        let started = tmp.path().join("pre-gate-started");
15238        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15239        runner.state.config.verify.gate =
15240            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15241        let home = crate::run::home();
15242        let owner =
15243            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15244        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15245            crate::cache::AcquireOutcome::Acquired(g) => g,
15246            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15247        };
15248        let mut gate = Box::pin(runner.run_gate(&winner));
15249        assert!(
15250            tokio::time::timeout(Duration::from_millis(150), &mut gate)
15251                .await
15252                .is_err()
15253        );
15254        assert!(!started.exists(), "pre-gate must wait for the cache too");
15255        // Simulate the wave that already owns the cache entering invoke.
15256        // It must obtain the editor reservation and finish without waiting
15257        // for the gate that is queued behind its cache.
15258        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15259        let editor = crate::repo_lock::SeatLease::checkout_with(
15260            &winner.worktree,
15261            true,
15262            crate::config::test_coordination().as_ref(),
15263            &wait,
15264        )
15265        .await
15266        .unwrap();
15267        drop(editor);
15268        guard.release();
15269        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15270            .await
15271            .unwrap()
15272            .unwrap();
15273        drop(gate);
15274        assert!(outcomes.iter().all(CommandOutcome::ok));
15275        assert!(started.exists());
15276    }
15277
15278    #[tokio::test]
15279    #[cfg(unix)]
15280    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15281        use std::os::fd::AsRawFd as _;
15282
15283        ask_test_home();
15284        let tmp = tempfile::tempdir().unwrap();
15285        let repo = tmp.path().join("repo");
15286        std::fs::create_dir_all(&repo).unwrap();
15287        init_repo(&repo);
15288        let common = repo.join(".git");
15289        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15290        let mut runner = runner_at(RunStatus::Prep);
15291        runner.state.repo = repo.clone();
15292        runner.state.config.coordination = crate::config::test_coordination();
15293        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15294        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15295        runner.state.config.graph.advise = false;
15296        runner.roles.implementers = vec![spec("implement")];
15297        runner.roles.judges = vec![spec("judge")];
15298        runner.prep().await.unwrap();
15299        let winner = runner.state.candidates[0].clone();
15300        // Emulate edits already left by a completed seat. The fixture CLI
15301        // only reports completion; it performs no repository operations.
15302        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15303        let completed = tmp.path().join("seat-completed");
15304        runner.roles.implementers[0].command = vec![
15305            "sh".into(),
15306            "-c".into(),
15307            format!("touch '{}'; echo completed", completed.display()),
15308        ];
15309        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15310        assert_eq!(
15311            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15312            0
15313        );
15314        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15315        let mut implement = Box::pin(runner.implement());
15316        tokio::select! {
15317            result = &mut implement => panic!("rescue must wait: {result:?}"),
15318            result = tokio::time::timeout(Duration::from_secs(10), async {
15319                while !completed.exists() {
15320                    tokio::time::sleep(Duration::from_millis(20)).await;
15321                }
15322            }) => result.unwrap(),
15323        }
15324        assert!(
15325            tokio::time::timeout(Duration::from_millis(200), &mut implement)
15326                .await
15327                .is_err()
15328        );
15329        assert_eq!(
15330            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15331            before
15332        );
15333        assert_eq!(
15334            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15335            0
15336        );
15337        tokio::time::timeout(Duration::from_secs(10), &mut implement)
15338            .await
15339            .unwrap()
15340            .unwrap();
15341        drop(implement);
15342        assert_ne!(
15343            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15344            before
15345        );
15346        assert!(git::is_clean(&winner.worktree).await.unwrap());
15347
15348        let started = tmp.path().join("pre-gate-started");
15349        let proceed = tmp.path().join("pre-gate-proceed");
15350        let checked = tmp.path().join("gate-ran");
15351        runner.state.config.verify.pre_gate = vec![format!(
15352            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15353            started.display(),
15354            proceed.display()
15355        )];
15356        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15357        assert_eq!(
15358            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15359            0
15360        );
15361        let mut gate = Box::pin(runner.run_gate(&winner));
15362        assert!(
15363            tokio::time::timeout(Duration::from_millis(200), &mut gate)
15364                .await
15365                .is_err()
15366        );
15367        assert!(!started.exists());
15368        assert!(!checked.exists());
15369        assert!(git::is_clean(&winner.worktree).await.unwrap());
15370        assert_eq!(
15371            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15372            0
15373        );
15374        tokio::select! {
15375            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15376            result = tokio::time::timeout(Duration::from_secs(10), async {
15377                while !started.exists() {
15378                    tokio::time::sleep(Duration::from_millis(20)).await;
15379                }
15380            }) => result.unwrap(),
15381        }
15382        assert_eq!(
15383            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15384            -1,
15385            "mechanical edits must hold the real lock"
15386        );
15387        std::fs::write(proceed, "ready").unwrap();
15388        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15389            .await
15390            .unwrap()
15391            .unwrap();
15392        drop(gate);
15393        assert!(outcomes.iter().all(CommandOutcome::ok));
15394        assert!(checked.exists());
15395        assert!(runner.state.pre_gate_commit.is_some());
15396        assert!(git::is_clean(&winner.worktree).await.unwrap());
15397        assert_eq!(
15398            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15399            0,
15400            "gate completion must release coordination"
15401        );
15402        drop(outsider);
15403        std::fs::remove_file(common.join("updater.lock")).unwrap();
15404        std::fs::create_dir(common.join("updater.lock")).unwrap();
15405        std::fs::remove_file(&started).unwrap();
15406        std::fs::remove_file(&checked).unwrap();
15407        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15408        let error = runner.run_gate(&winner).await.unwrap_err();
15409        assert!(
15410            error
15411                .to_string()
15412                .contains("acquire mandatory repository lock")
15413        );
15414        assert!(
15415            !started.exists(),
15416            "failed acquisition must prevent pre-gate"
15417        );
15418        assert!(!checked.exists(), "failed acquisition must prevent gate");
15419        assert!(git::is_clean(&winner.worktree).await.unwrap());
15420        assert_eq!(
15421            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15422            head
15423        );
15424    }
15425
15426    #[tokio::test]
15427    #[cfg(unix)]
15428    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15429        use std::os::fd::AsRawFd as _;
15430        use std::time::Duration;
15431
15432        ask_test_home();
15433        let tmp = tempfile::tempdir().unwrap();
15434        let repo = tmp.path().join("repo");
15435        std::fs::create_dir_all(&repo).unwrap();
15436        init_repo(&repo);
15437        let common = repo.join(".git");
15438        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15439        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15440        assert_eq!(
15441            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15442            0
15443        );
15444        let mut runner = runner_at(RunStatus::Prep);
15445        runner.state.repo = repo.clone();
15446        runner.state.config.coordination = crate::config::test_coordination();
15447        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15448        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15449        runner.state.config.graph.advise = true;
15450        runner.state.config.graph.advisors = 1;
15451        runner.roles.implementers = vec![spec("implement")];
15452        runner.roles.judges = vec![spec("judge")];
15453        let root = runner.state.worktree_root();
15454        let started = tmp.path().join("setup-started");
15455        let proceed = tmp.path().join("setup-proceed");
15456        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15457            run: Some(format!(
15458                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15459                started.display(),
15460                proceed.display()
15461            )),
15462            ..Default::default()
15463        }];
15464        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15465        let config = git::git(&repo, &["config", "--local", "--list"])
15466            .await
15467            .unwrap();
15468        let mut prep = Box::pin(runner.prep());
15469        assert!(
15470            tokio::time::timeout(Duration::from_millis(200), &mut prep)
15471                .await
15472                .is_err()
15473        );
15474        assert!(
15475            !root.exists(),
15476            "contention must prevent even checkout creation"
15477        );
15478        assert!(!started.exists(), "contention must prevent setup");
15479        assert_eq!(
15480            git::git(&repo, &["branch", "--list"]).await.unwrap(),
15481            branches
15482        );
15483        assert_eq!(
15484            git::git(&repo, &["config", "--local", "--list"])
15485                .await
15486                .unwrap(),
15487            config
15488        );
15489        drop(lock);
15490        tokio::select! {
15491            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15492            result = tokio::time::timeout(Duration::from_secs(10), async {
15493                while !started.exists() {
15494                    tokio::time::sleep(Duration::from_millis(20)).await;
15495                }
15496            }) => result.unwrap(),
15497        }
15498        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15499        assert_eq!(
15500            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15501            -1
15502        );
15503        assert!(
15504            tokio::time::timeout(
15505                Duration::from_millis(100),
15506                crate::repo_lock::SeatLease::for_checkout(
15507                    &repo,
15508                    true,
15509                    crate::config::test_coordination().as_ref()
15510                )
15511            )
15512            .await
15513            .is_err(),
15514            "preparation must also exclude local editors"
15515        );
15516        std::fs::write(proceed, "ready").unwrap();
15517        tokio::time::timeout(Duration::from_secs(10), &mut prep)
15518            .await
15519            .unwrap()
15520            .unwrap();
15521        drop(prep);
15522        assert_eq!(runner.state.prep_complete, Some(true));
15523        assert!(root.join("judge-1").exists());
15524        assert!(root.join("advisor-1").exists());
15525        assert_eq!(
15526            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15527            0
15528        );
15529    }
15530
15531    #[tokio::test]
15532    #[cfg(unix)]
15533    async fn preparation_lock_failure_prevents_repository_changes() {
15534        ask_test_home();
15535        let tmp = tempfile::tempdir().unwrap();
15536        let repo = tmp.path().join("repo");
15537        std::fs::create_dir_all(&repo).unwrap();
15538        init_repo(&repo);
15539        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15540        // A directory at the required lock path makes opening the real lock
15541        // fail, rather than merely exercising an invalid registration.
15542        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15543        let mut runner = runner_at(RunStatus::Prep);
15544        runner.state.repo = repo.clone();
15545        runner.state.config.coordination = crate::config::test_coordination();
15546        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15547        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15548        runner.roles.implementers = vec![spec("implement")];
15549        let error = runner.prep().await.unwrap_err();
15550        assert!(
15551            error
15552                .to_string()
15553                .contains("acquire mandatory repository lock")
15554        );
15555        assert!(!runner.state.worktree_root().exists());
15556        assert!(runner.state.candidates.is_empty());
15557        assert!(!repo.join(".git/config.worktree").exists());
15558        assert_eq!(
15559            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15560            "* main"
15561        );
15562    }
15563
15564    #[tokio::test]
15565    async fn preparation_records_creation_intent_before_git_can_fail() {
15566        ask_test_home();
15567        let tmp = tempfile::tempdir().unwrap();
15568        let repo = tmp.path().join("repo");
15569        std::fs::create_dir_all(&repo).unwrap();
15570        init_repo(&repo);
15571        let mut runner = runner_at(RunStatus::Prep);
15572        runner.state.repo = repo.clone();
15573        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15574        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15575        runner.state.config.blind.commit_msg_hook = false;
15576        runner.roles.implementers = vec![spec("one")];
15577        let label = blind::assign_labels(1, runner.state.seed)[0];
15578        let branch = runner.state.branch_for(label);
15579        // A conflicting branch makes Git refuse creation before a candidate
15580        // can be registered. The durable intent must still identify it.
15581        git::git(&repo, &["branch", &branch]).await.unwrap();
15582        assert!(runner.prep().await.is_err());
15583        let saved = RunState::load(&runner.state.id).unwrap();
15584        assert!(saved.candidates.is_empty());
15585        let event = saved
15586            .events
15587            .iter()
15588            .find(|e| e.node == "prep-intent")
15589            .unwrap();
15590        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15591        assert_eq!(intent["branch"], branch);
15592        assert_eq!(intent["start"], runner.state.base_commit);
15593        assert_eq!(
15594            intent["worktree"],
15595            runner
15596                .state
15597                .worktree_root()
15598                .join(format!("cand-{label}"))
15599                .to_string_lossy()
15600                .as_ref()
15601        );
15602        assert_eq!(intent["picks"], serde_json::json!([]));
15603    }
15604
15605    #[tokio::test]
15606    async fn invalid_pinned_sources_create_no_partial_worktrees() {
15607        ask_test_home();
15608        let tmp = tempfile::tempdir().unwrap();
15609        let repo = tmp.path().join("repo");
15610        std::fs::create_dir_all(&repo).unwrap();
15611        init_repo(&repo);
15612        let mut runner = runner_at(RunStatus::Prep);
15613        runner.state.repo = repo.clone();
15614        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15615        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15616        runner.state.sources.pick_commits.push(refs::Input {
15617            token: "missing".into(),
15618            sha: "0".repeat(40),
15619            provenance: refs::Provenance::ExplicitCli,
15620        });
15621        assert!(runner.prep().await.is_err());
15622        assert!(runner.state.candidates.is_empty());
15623        assert!(!runner.state.worktree_root().exists());
15624    }
15625
15626    #[tokio::test]
15627    async fn prose_and_legacy_inference_never_import_into_candidates() {
15628        ask_test_home();
15629        let tmp = tempfile::tempdir().unwrap();
15630        let repo = tmp.path().join("repo");
15631        std::fs::create_dir_all(&repo).unwrap();
15632        init_repo(&repo);
15633        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15634        git::git(&repo, &["checkout", "-b", "magi/history/A"])
15635            .await
15636            .unwrap();
15637        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15638        git::git(&repo, &["add", "."]).await.unwrap();
15639        git::git(&repo, &["commit", "-m", "historical implementation"])
15640            .await
15641            .unwrap();
15642        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15643        git::git(&repo, &["checkout", "main"]).await.unwrap();
15644        for instruction in [
15645            format!("Example: magi/history/A at {source}"),
15646            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15647            format!("Never import magi/history/A or {source}"),
15648            format!("Conversation answer: I mentioned {source} only as background"),
15649        ] {
15650            let mut runner = runner_at(RunStatus::Prep);
15651            runner.state.repo = repo.clone();
15652            runner.state.base_commit = base.clone();
15653            runner.state.instruction = instruction.clone();
15654            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15655            runner.state.config.blind.commit_msg_hook = false;
15656            runner.state.seeds = vec![refs::Seed {
15657                token: "magi/history/A".into(),
15658                sha: source.clone(),
15659                kind: refs::SeedKind::Unmerged,
15660                branch: true,
15661                detail: String::new(),
15662            }];
15663            runner.roles.implementers = vec![spec("one")];
15664            assert_eq!(seeded_instruction(&runner.state), instruction);
15665            // Model an old record with inferred seeds but no structured input field.
15666            let mut legacy = serde_json::to_value(&runner.state).unwrap();
15667            legacy.as_object_mut().unwrap().remove("sources");
15668            runner.state = serde_json::from_value(legacy).unwrap();
15669            assert_eq!(runner.state.sources, refs::Sources::default());
15670            runner.state.save().unwrap();
15671            runner.state = RunState::load(&runner.state.id).unwrap();
15672            runner.prep().await.unwrap();
15673            for candidate in &runner.state.candidates {
15674                assert_eq!(
15675                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15676                    base
15677                );
15678                assert!(!candidate.worktree.join("unwanted").exists());
15679            }
15680        }
15681    }
15682
15683    #[tokio::test]
15684    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15685        ask_test_home();
15686        let tmp = tempfile::tempdir().unwrap();
15687        let repo = tmp.path().join("repo");
15688        std::fs::create_dir_all(&repo).unwrap();
15689        init_repo(&repo);
15690        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15691        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15692        std::fs::write(repo.join("referenced"), "required commit").unwrap();
15693        git::git(&repo, &["add", "."]).await.unwrap();
15694        git::git(&repo, &["commit", "-m", "required seed"])
15695            .await
15696            .unwrap();
15697        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15698        git::git(&repo, &["checkout", "main"]).await.unwrap();
15699        let mut runner = runner_at(RunStatus::Prep);
15700        runner.state.repo = repo.clone();
15701        runner.state.base_commit = base.clone();
15702        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15703        runner.state.config.graph.advise = false;
15704        runner.state.config.blind.commit_msg_hook = false;
15705        runner.state.sources =
15706            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15707                .await
15708                .unwrap();
15709        runner.roles.implementers = vec![spec("one"), spec("two")];
15710        runner.roles.judges = vec![spec("judge")];
15711        let label = blind::assign_labels(2, runner.state.seed)[0];
15712        let branch = runner.state.branch_for(label);
15713        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15714        git::worktree_add_branch(&repo, &worktree, &branch, &base)
15715            .await
15716            .unwrap();
15717        runner.state.candidates.push(Candidate {
15718            index: 0,
15719            label,
15720            agent: "one".into(),
15721            branch,
15722            worktree,
15723            summary: String::new(),
15724            stat: String::new(),
15725            files: 0,
15726            commits: 0,
15727            empty: false,
15728            failed: None,
15729            verified_noop: None,
15730            duration_ms: 0,
15731            folded: false,
15732        });
15733        runner.state.save().unwrap();
15734        // Reload the checkpoint immediately before its first cherry-pick.
15735        runner.state = RunState::load(&runner.state.id).unwrap();
15736        assert_eq!(runner.state.prep_complete, Some(false));
15737        runner.prep().await.unwrap();
15738        assert_eq!(runner.state.prep_complete, Some(true));
15739        assert_eq!(runner.state.candidates.len(), 2);
15740        for candidate in &runner.state.candidates {
15741            assert_eq!(
15742                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15743                git::tree_of(&repo, &seed).await.unwrap()
15744            );
15745            assert_eq!(
15746                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15747                "required commit"
15748            );
15749        }
15750        assert!(runner.state.worktree_root().join("judge-1").exists());
15751        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15752        runner.prep().await.unwrap();
15753        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15754
15755        runner.state.prep_complete = Some(false);
15756        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15757            run: Some(format!(
15758                "test -f '{}'",
15759                tmp.path().join("dependency-ready").display()
15760            )),
15761            ..Default::default()
15762        }];
15763        assert!(runner.prep().await.is_err());
15764        assert!(runner.state.prep_failure.is_some());
15765        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15766        runner.state = RunState::load(&runner.state.id).unwrap();
15767        assert!(
15768            runner.prep().await.is_err(),
15769            "correcting the cause alone must not auto-retry"
15770        );
15771        let failed_at = runner.state.events.last().unwrap().at;
15772        assert!(
15773            runner.retry_preparation(failed_at).is_err(),
15774            "old approvals cannot unlock the hold"
15775        );
15776        let mut task = crate::queue::Task::new(
15777            "retry".into(),
15778            "retry setup".into(),
15779            repo.clone(),
15780            crate::queue::Source::Human,
15781        );
15782        task.start(runner.state.id.clone());
15783        task.hold_machine(Some("setup failed".into()));
15784        task.release_operator("cli", None);
15785        runner
15786            .retry_preparation(task.resumption.as_ref().unwrap().at)
15787            .unwrap();
15788        runner.prep().await.unwrap();
15789        assert!(runner.state.prep_failure.is_none());
15790        assert_eq!(runner.state.prep_complete, Some(true));
15791        assert_eq!(runner.state.candidates.len(), 2);
15792        assert!(runner.state.worktree_root().join("judge-1").exists());
15793        for candidate in &runner.state.candidates {
15794            assert_eq!(
15795                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15796                git::tree_of(&repo, &seed).await.unwrap()
15797            );
15798        }
15799    }
15800
15801    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15802        let mut heads = Vec::new();
15803        for candidate in candidates {
15804            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15805        }
15806        heads
15807    }
15808
15809    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15810        ask_test_home();
15811        let tmp = tempfile::tempdir().unwrap();
15812        let repo = tmp.path().join("repo");
15813        std::fs::create_dir(&repo).unwrap();
15814        init_repo(&repo);
15815        let mut runner = runner_at(RunStatus::Reviewing);
15816        runner.state.repo = repo.clone();
15817        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15818        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15819        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15820            let branch = format!("candidate-{label}");
15821            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15822            git::worktree_add_branch(&repo, &worktree, &branch, "main")
15823                .await
15824                .unwrap();
15825            if label == 'B' {
15826                std::fs::write(worktree.join("change"), "winner").unwrap();
15827                git::rescue_commit(&worktree, "candidate change")
15828                    .await
15829                    .unwrap();
15830            }
15831            runner.state.candidates.push(Candidate {
15832                index,
15833                label,
15834                agent: "alpha".into(),
15835                branch,
15836                worktree,
15837                summary: "completed reply".into(),
15838                stat: String::new(),
15839                files: usize::from(label == 'B'),
15840                commits: usize::from(label == 'B'),
15841                empty: label == 'A',
15842                failed: None,
15843                verified_noop: None,
15844                duration_ms: 1,
15845                folded: false,
15846            });
15847        }
15848        runner.state.judge_skipped = true;
15849        runner.state.tally = Some(Tally {
15850            first_choice: BTreeMap::new(),
15851            borda: BTreeMap::new(),
15852            winner: 'B',
15853            rankings: 0,
15854            unanimous_initial: false,
15855            deliberated: false,
15856            changed_votes: 0,
15857            unanimous_final: false,
15858            tie_break: None,
15859            judges: 0,
15860            present: 0,
15861            quorum: 0,
15862            met_quorum: true,
15863            uncontested: Some("only B produced a change".into()),
15864        });
15865        runner.state.save().unwrap();
15866        (tmp, runner)
15867    }
15868
15869    #[tokio::test]
15870    #[cfg(unix)]
15871    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15872        for (solo, failed_checkpoint) in
15873            [(true, false), (false, false), (true, true), (false, true)]
15874        {
15875            let (_tmp, mut runner) = preservation_resume_fixture().await;
15876            if solo {
15877                runner.state.candidates.retain(|c| c.label == 'A');
15878            }
15879            runner.state.candidates[0].empty = false;
15880            runner.state.candidates[0].summary.clear();
15881            runner.state.tally = None;
15882            runner.state.judge_skipped = false;
15883            runner.state.status = RunStatus::Implementing;
15884            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15885            let mut implementer = spec("alpha");
15886            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15887            runner.roles.implementers = vec![implementer];
15888            let patch = runner.state.dir().join("artifacts/cand-A.patch");
15889            if failed_checkpoint {
15890                std::fs::create_dir_all(&patch).unwrap();
15891            }
15892            runner.state.save().unwrap();
15893            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15894            drop(pressure);
15895            runner.state = RunState::load(&runner.state.id).unwrap();
15896            assert_eq!(runner.state.status, RunStatus::Blocked);
15897            assert!(runner.state.interrupted_implementations.contains(&'A'));
15898            assert_eq!(runner.state.candidates[0].commits, 0);
15899            assert!(!runner.state.candidates[0].empty);
15900            assert!(runner.state.candidates[0].failed.is_none());
15901            if failed_checkpoint {
15902                assert!(runner.state.candidate_preservation.contains_key(&'A'));
15903                std::fs::remove_dir(patch).unwrap();
15904            }
15905            runner.state.resource_block = None;
15906            runner.state.config.disk.min_free_bytes = 0;
15907            runner.roles.implementers[0].command = vec![
15908                "sh".into(),
15909                "-c".into(),
15910                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15911            ];
15912            runner.implement().await.unwrap();
15913            assert!(runner.state.interrupted_implementations.is_empty());
15914            assert!(runner.state.candidate_preservation.is_empty());
15915            assert_eq!(runner.state.candidates[0].commits, 1);
15916            assert!(!runner.state.candidates[0].empty);
15917            assert!(
15918                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15919                    .unwrap()
15920                    .contains("retried empty interrupted seat")
15921            );
15922            if !solo {
15923                assert_eq!(runner.state.candidates[1].commits, 1);
15924            }
15925        }
15926    }
15927
15928    #[tokio::test]
15929    #[cfg(unix)]
15930    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15931        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15932            let (_tmp, mut runner) = preservation_resume_fixture().await;
15933            runner.state.candidates.retain(|c| c.label == 'A');
15934            runner.state.candidates[0].empty = false;
15935            runner.state.candidates[0].summary.clear();
15936            runner.state.tally = None;
15937            runner.state.judge_skipped = false;
15938            runner.state.status = RunStatus::Implementing;
15939            // Force only the real post-spawn watchdog, without failing the
15940            // separate admission check. Existing edits predate its measurement.
15941            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15942            let wt = runner.state.candidates[0].worktree.clone();
15943            std::fs::write(
15944                wt.join("interrupted-source"),
15945                "retain implementation work\n",
15946            )
15947            .unwrap();
15948            if committed {
15949                git::rescue_commit(&wt, "implementation checkpoint")
15950                    .await
15951                    .unwrap();
15952            }
15953            let lock = git::git(
15954                &wt,
15955                &[
15956                    "rev-parse",
15957                    "--path-format=absolute",
15958                    "--git-path",
15959                    "index.lock",
15960                ],
15961            )
15962            .await
15963            .unwrap();
15964            if block_rescue {
15965                std::fs::write(&lock, "fixture-owned lock").unwrap();
15966            }
15967            let mut implementer = spec("alpha");
15968            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15969            runner.roles.implementers = vec![implementer];
15970            runner.state.save().unwrap();
15971            let result = runner.implement().await;
15972            drop(pressure);
15973            assert_eq!(result.is_err(), block_rescue);
15974            assert_eq!(
15975                runner.state.resource_block.as_ref().unwrap().kind,
15976                "disk pressure"
15977            );
15978            assert!(
15979                runner
15980                    .state
15981                    .events
15982                    .iter()
15983                    .any(|e| e.message.contains("agent execution interrupted"))
15984            );
15985            assert!(
15986                runner
15987                    .state
15988                    .dir()
15989                    .join("artifacts/candidate-A-recovery/staged.patch")
15990                    .exists()
15991            );
15992            if block_rescue {
15993                let saved = RunState::load(&runner.state.id).unwrap();
15994                assert!(saved.candidate_preservation.contains_key(&'A'));
15995                assert!(!saved.candidates[0].empty);
15996                assert_eq!(
15997                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15998                    "retain implementation work\n"
15999                );
16000                std::fs::remove_file(lock).unwrap();
16001                runner.state = saved;
16002            } else {
16003                assert_eq!(runner.state.status, RunStatus::Blocked);
16004                assert_eq!(runner.state.candidates[0].commits, 1);
16005            }
16006            // Recovery must save the candidate before deciding whether another
16007            // implementation is needed. No implementer is available on retry.
16008            runner.roles.implementers.clear();
16009            runner.state.config.disk.min_free_bytes = 0;
16010            runner.state.resource_block = None;
16011            runner.implement().await.unwrap();
16012            assert!(runner.state.candidate_preservation.is_empty());
16013            assert_eq!(runner.state.candidates[0].commits, 1);
16014            assert!(!runner.state.candidates[0].empty);
16015            assert!(git::is_clean(&wt).await.unwrap());
16016            assert!(
16017                crate::run::read_artifact(&runner.state, "cand-A.patch")
16018                    .unwrap()
16019                    .contains("retain implementation work")
16020            );
16021        }
16022    }
16023
16024    #[tokio::test]
16025    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16026        let (_tmp, mut runner) = preservation_resume_fixture().await;
16027        // Schema 15 already confirmed the unfolded no-change result.
16028        runner.implement().await.unwrap();
16029        assert!(runner.state.candidate_preservation.is_empty());
16030        assert!(
16031            !runner
16032                .state
16033                .dir()
16034                .join("artifacts/candidate-A-recovery")
16035                .exists()
16036        );
16037        runner.fold_losers().await.unwrap();
16038        assert!(!runner.state.candidates[0].worktree.exists());
16039        for schema in [14, crate::run::SCHEMA] {
16040            runner.state.schema = schema;
16041            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16042            runner.state.save().unwrap();
16043            runner.state = RunState::load(&runner.state.id).unwrap();
16044            runner.implement().await.unwrap();
16045            assert!(runner.state.candidate_preservation.is_empty());
16046            assert!(runner.state.candidates[0].folded);
16047            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16048        }
16049    }
16050
16051    #[tokio::test]
16052    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16053        let (_tmp, mut runner) = preservation_resume_fixture().await;
16054        runner.state.candidates.retain(|c| c.label == 'A');
16055        runner.state.tally = None;
16056        runner.state.judge_skipped = false;
16057        let candidate = &mut runner.state.candidates[0];
16058        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16059        candidate.failed = Some("deployment validation unavailable".into());
16060        std::fs::write(
16061            candidate.worktree.join("repair"),
16062            "future launcher policy\n",
16063        )
16064        .unwrap();
16065        runner
16066            .state
16067            .candidate_preservation
16068            .insert('A', "awaiting persistence".into());
16069        runner.persist_candidates().await.unwrap();
16070        let candidate = &runner.state.candidates[0];
16071        assert!(!candidate.empty);
16072        assert_eq!(candidate.commits, 1);
16073        assert_eq!(candidate.files, 1);
16074        assert!(candidate.failed.is_none());
16075        assert!(candidate.summary.contains("EPERM"));
16076        assert!(
16077            crate::run::read_artifact(&runner.state, "cand-A.patch")
16078                .unwrap()
16079                .contains("future launcher policy")
16080        );
16081        assert!(runner.state.candidate_preservation.is_empty());
16082    }
16083
16084    #[tokio::test]
16085    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16086        let (_tmp, mut runner) = preservation_resume_fixture().await;
16087        let wt = runner.state.candidates[0].worktree.clone();
16088        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16089        runner.state.schema = 14;
16090        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16091        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16092        let tally = serde_json::to_value(&runner.state.tally).unwrap();
16093        for _ in 0..2 {
16094            assert!(runner.implement().await.is_err());
16095            assert_eq!(runner.state.status, RunStatus::Blocked);
16096            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16097            assert!(
16098                runner.state.candidates[0].empty,
16099                "old verdict must remain unusable"
16100            );
16101            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16102            assert!(runner.state.judge_skipped);
16103            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16104            assert!(
16105                crate::run::read_artifact(&runner.state, "cand-A.patch")
16106                    .unwrap()
16107                    .contains("owned edits")
16108            );
16109            assert!(wt.exists());
16110            assert_eq!(
16111                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16112                    .await
16113                    .unwrap(),
16114                1
16115            );
16116            runner.state = RunState::load(&runner.state.id).unwrap();
16117        }
16118    }
16119
16120    #[tokio::test]
16121    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16122        let (_tmp, mut runner) = preservation_resume_fixture().await;
16123        let wt = runner.state.candidates[0].worktree.clone();
16124        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16125        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16126        let index_lock = PathBuf::from(
16127            git::git(
16128                &wt,
16129                &[
16130                    "rev-parse",
16131                    "--path-format=absolute",
16132                    "--git-path",
16133                    "index.lock",
16134                ],
16135            )
16136            .await
16137            .unwrap(),
16138        );
16139        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16140        runner.state.schema = 14;
16141        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16142        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16143        assert!(runner.implement().await.is_err());
16144        assert!(runner.after_implement().is_err());
16145        assert!(runner.fold_losers().await.is_err());
16146        assert_eq!(
16147            std::fs::read_to_string(wt.join("recovered")).unwrap(),
16148            "failed seat edits"
16149        );
16150        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16151        runner.state = RunState::load(&runner.state.id).unwrap();
16152        std::fs::remove_file(index_lock).unwrap();
16153        // Recovery finds changes that the old uncontested result never judged.
16154        assert!(runner.implement().await.is_err());
16155        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16156        assert!(runner.state.candidates[0].empty);
16157        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16158        assert!(
16159            crate::run::read_artifact(&runner.state, "cand-A.patch")
16160                .unwrap()
16161                .contains("failed seat edits")
16162        );
16163        assert!(wt.exists());
16164    }
16165
16166    #[tokio::test]
16167    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16168        let (_tmp, mut runner) = preservation_resume_fixture().await;
16169        let repo = runner.state.repo.clone();
16170        let common = git::git(
16171            &repo,
16172            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16173        )
16174        .await
16175        .unwrap();
16176        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16177        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16178        assert!(runner.fold_losers().await.is_err());
16179        assert!(!runner.state.candidates[0].worktree.exists());
16180        assert!(!runner.state.candidates[0].folded);
16181        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16182        runner.state = RunState::load(&runner.state.id).unwrap();
16183        std::fs::remove_file(branch_lock).unwrap();
16184        let judge = runner.state.worktree_root().join("judge-1");
16185        git::worktree_add_detached(&repo, &judge, "main")
16186            .await
16187            .unwrap();
16188        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16189        runner.roles.judges = vec![spec("judge")];
16190        assert!(runner.fold_losers().await.is_err());
16191        runner.state = RunState::load(&runner.state.id).unwrap();
16192        assert!(
16193            runner.state.candidates[0].folded,
16194            "candidate progress saved before judge failure"
16195        );
16196        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16197        assert!(judge.join("uncommitted").exists());
16198        std::fs::remove_file(judge.join("uncommitted")).unwrap();
16199        runner.fold_losers().await.unwrap();
16200        assert!(!judge.exists());
16201        runner.fold_losers().await.unwrap();
16202    }
16203
16204    #[tokio::test]
16205    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16206        let (_tmp, mut runner) = preservation_resume_fixture().await;
16207        runner.state.candidates.retain(|c| c.label == 'B');
16208        runner.state.tally = None;
16209        runner.state.judge_skipped = false;
16210        runner.state.prep_complete = Some(true);
16211        runner.state.config.graph.advise = false;
16212        runner.state.config.graph.selected_implementer = Some("alpha".into());
16213        let wt = runner.state.candidates[0].worktree.clone();
16214        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16215        runner
16216            .state
16217            .candidate_preservation
16218            .insert('B', "awaiting persistence".into());
16219        let index_lock = git::git(
16220            &wt,
16221            &[
16222                "rev-parse",
16223                "--path-format=absolute",
16224                "--git-path",
16225                "index.lock",
16226            ],
16227        )
16228        .await
16229        .unwrap();
16230        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16231        runner.state.save().unwrap();
16232        assert!(runner.execute_graph().await.is_err());
16233        assert_eq!(runner.state.status, RunStatus::Blocked);
16234        assert!(runner.state.candidate_preservation.contains_key(&'B'));
16235        std::fs::remove_file(index_lock).unwrap();
16236        // Park at the next graph boundary: this tests the real selected-seat
16237        // hold guard without dispatching reviewers or publishing anything.
16238        for _ in 0..2 {
16239            runner.state = RunState::load(&runner.state.id).unwrap();
16240            runner.pause.park();
16241            runner.execute_graph().await.unwrap();
16242            assert_eq!(runner.state.status, RunStatus::Implementing);
16243            assert!(runner.state.candidate_preservation.is_empty());
16244            assert_eq!(runner.state.candidates[0].commits, 2);
16245            assert!(runner.state.tally.is_none());
16246        }
16247        runner.implement().await.unwrap();
16248        assert_eq!(runner.state.status, RunStatus::Judging);
16249        // A genuine selected-implementer hold must still require release.
16250        runner.state.status = RunStatus::Blocked;
16251        runner.state.candidates[0].failed = Some("implementation failed".into());
16252        runner.state.save().unwrap();
16253        let error = runner.execute_graph().await.unwrap_err();
16254        assert!(
16255            error
16256                .to_string()
16257                .contains("selected implementation is held")
16258        );
16259    }
16260
16261    #[tokio::test]
16262    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16263        let (_tmp, mut runner) = preservation_resume_fixture().await;
16264        let repo = runner.state.repo.clone();
16265        let candidate = runner.state.candidates[0].clone();
16266        // Remove only the disposable fixture directory, leaving Git's linked
16267        // worktree registration and branch intact, as in a vanished checkout.
16268        let registered = git::worktree_holding(&repo, &candidate.branch)
16269            .await
16270            .unwrap()
16271            .unwrap();
16272        assert_eq!(
16273            registered.canonicalize().unwrap(),
16274            candidate.worktree.canonicalize().unwrap()
16275        );
16276        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16277        assert_eq!(
16278            git::worktree_holding(&repo, &candidate.branch)
16279                .await
16280                .unwrap(),
16281            Some(registered)
16282        );
16283        assert!(
16284            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16285            "the stale registration initially keeps the branch checked out"
16286        );
16287        runner.fold_losers().await.unwrap();
16288        assert!(runner.state.candidates[0].folded);
16289        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16290        assert_eq!(
16291            git::worktree_holding(&repo, &candidate.branch)
16292                .await
16293                .unwrap(),
16294            None
16295        );
16296        runner.state = RunState::load(&runner.state.id).unwrap();
16297        runner.fold_losers().await.unwrap();
16298    }
16299
16300    #[tokio::test]
16301    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16302        let (_tmp, mut runner) = preservation_resume_fixture().await;
16303        let repo = runner.state.repo.clone();
16304        let candidate = runner.state.candidates[0].clone();
16305        let winner = runner.state.candidates[1].clone();
16306        let registered = git::worktree_holding(&repo, &candidate.branch)
16307            .await
16308            .unwrap()
16309            .unwrap();
16310        assert_eq!(
16311            registered.canonicalize().unwrap(),
16312            candidate.worktree.canonicalize().unwrap()
16313        );
16314        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16315        assert_eq!(
16316            git::worktree_holding(&repo, &candidate.branch)
16317                .await
16318                .unwrap(),
16319            Some(registered)
16320        );
16321        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16322
16323        for _ in 0..2 {
16324            fold_run(&mut runner.state, false, &crate::run::home())
16325                .await
16326                .unwrap();
16327            runner.state = RunState::load(&runner.state.id).unwrap();
16328            assert!(runner.state.candidates[0].folded);
16329            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16330            assert_eq!(
16331                git::worktree_holding(&repo, &candidate.branch)
16332                    .await
16333                    .unwrap(),
16334                None
16335            );
16336            assert!(!runner.state.candidates[1].folded);
16337            assert!(winner.worktree.exists());
16338            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16339        }
16340    }
16341
16342    #[tokio::test]
16343    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16344        ask_test_home();
16345        let tmp = tempfile::tempdir().unwrap();
16346        let repo = tmp.path().join("repo");
16347        std::fs::create_dir(&repo).unwrap();
16348        init_repo(&repo);
16349        let mut runner = runner_at(RunStatus::Implementing);
16350        runner.state.repo = repo.clone();
16351        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16352        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16353        for (index, label) in ['A', 'B'].into_iter().enumerate() {
16354            let branch = format!("candidate-{label}");
16355            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16356            git::worktree_add_branch(&repo, &wt, &branch, "main")
16357                .await
16358                .unwrap();
16359            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16360            git::git(&wt, &["add", "README.md"]).await.unwrap();
16361            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16362            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16363            runner.state.candidates.push(Candidate {
16364                index,
16365                label,
16366                agent: "alpha".to_owned(),
16367                branch,
16368                worktree: wt,
16369                summary: "completed implementation".to_owned(),
16370                stat: String::new(),
16371                files: 0,
16372                commits: 0,
16373                empty: false,
16374                failed: None,
16375                verified_noop: None,
16376                duration_ms: 42,
16377                folded: false,
16378            });
16379            runner
16380                .state
16381                .candidate_preservation
16382                .insert(label, "awaiting persistence".to_owned());
16383        }
16384        let wt = runner.state.candidates[0].worktree.clone();
16385        let index_lock = git::git(
16386            &wt,
16387            &[
16388                "rev-parse",
16389                "--path-format=absolute",
16390                "--git-path",
16391                "index.lock",
16392            ],
16393        )
16394        .await
16395        .unwrap();
16396        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16397        runner.state.save().unwrap();
16398        assert!(runner.persist_candidates().await.is_err());
16399        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16400        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16401        assert_eq!(
16402            runner.state.candidates[1].commits, 1,
16403            "other results are still collected"
16404        );
16405        assert!(
16406            !runner.state.candidates[0].empty,
16407            "failed rescue is not no change"
16408        );
16409        assert_eq!(
16410            runner.state.candidates[0].summary,
16411            "completed implementation"
16412        );
16413        assert!(runner.after_implement().is_err());
16414        let home = tmp.path().join("fold-home");
16415        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16416        assert!(wt.join("new-file").exists());
16417        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16418        assert!(recovery.join("staged.patch").exists());
16419        assert!(recovery.join("unstaged.patch").exists());
16420        assert_eq!(
16421            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16422            "untracked edits\n"
16423        );
16424        runner.state.begin_pass();
16425        runner.state.end_pass_parked(ParkCause::Upgrade {
16426            from: "1.0".into(),
16427            to: Some("1.1".into()),
16428        });
16429        runner.state.save().unwrap();
16430        runner.state = RunState::load(&runner.state.id).unwrap();
16431        assert!(matches!(
16432            runner.state.passes.last().unwrap().end,
16433            Some(crate::run::PassEnd::Parked {
16434                cause: ParkCause::Upgrade { .. }
16435            })
16436        ));
16437        std::fs::remove_file(index_lock).unwrap();
16438        // No implementer is configured: a persistence retry cannot dispatch a seat.
16439        runner.implement().await.unwrap();
16440        assert!(runner.state.candidate_preservation.is_empty());
16441        assert!(
16442            runner
16443                .state
16444                .candidates
16445                .iter()
16446                .all(|c| c.commits == 1 && !c.empty)
16447        );
16448        assert!(git::is_clean(&wt).await.unwrap());
16449        assert!(
16450            crate::run::read_artifact(&runner.state, "cand-A.patch")
16451                .unwrap()
16452                .contains("later tracked edits")
16453        );
16454    }
16455
16456    #[tokio::test]
16457    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16458        ask_test_home();
16459        let tmp = tempfile::tempdir().unwrap();
16460        let repo = tmp.path().join("repo");
16461        std::fs::create_dir(&repo).unwrap();
16462        init_repo(&repo);
16463        let wt = tmp.path().join("candidate");
16464        git::worktree_add_branch(&repo, &wt, "candidate", "main")
16465            .await
16466            .unwrap();
16467        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16468        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16469        assert!(wt.join("uncommitted").exists());
16470        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16471        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16472        let mut runner = runner_at(RunStatus::Implementing);
16473        runner.state.repo = repo.clone();
16474        runner.state.base_commit = "missing-base".to_owned();
16475        runner.state.candidates.push(Candidate {
16476            index: 0,
16477            label: 'A',
16478            agent: "alpha".into(),
16479            branch: "candidate".into(),
16480            worktree: wt.clone(),
16481            summary: "completed".into(),
16482            stat: String::new(),
16483            files: 0,
16484            commits: 0,
16485            empty: false,
16486            failed: None,
16487            verified_noop: None,
16488            duration_ms: 1,
16489            folded: false,
16490        });
16491        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16492        assert!(
16493            fold_run(&mut runner.state, true, &tmp.path().join("home"))
16494                .await
16495                .is_err()
16496        );
16497        assert!(!runner.state.candidates[0].folded);
16498        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16499        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16500        runner
16501            .state
16502            .candidate_preservation
16503            .insert('A', "pending".into());
16504        assert!(runner.persist_candidates().await.is_err());
16505        assert!(!runner.state.candidates[0].empty);
16506        assert!(!runner.state.candidates[0].folded);
16507        assert!(wt.exists());
16508        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16509        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16510        std::fs::create_dir_all(&patch_path).unwrap();
16511        assert!(
16512            runner.persist_candidates().await.is_err(),
16513            "artifact failure stays pending"
16514        );
16515        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16516        assert!(!runner.state.candidates[0].empty);
16517        std::fs::remove_dir(patch_path).unwrap();
16518        runner.persist_candidates().await.unwrap();
16519        assert!(
16520            runner.state.candidates[0].empty,
16521            "only successful inspection confirms no change"
16522        );
16523    }
16524
16525    fn init_repo(dir: &Path) {
16526        let run = |args: &[&str]| {
16527            let out = std::process::Command::new("git")
16528                .args(args)
16529                .current_dir(dir)
16530                .quiet()
16531                .output()
16532                .expect("spawn git");
16533            assert!(
16534                out.status.success(),
16535                "git {args:?} failed: {}",
16536                String::from_utf8_lossy(&out.stderr)
16537            );
16538        };
16539        run(&["init", "-b", "main"]);
16540        run(&["config", "user.name", "magi test"]);
16541        run(&["config", "user.email", "magi@example.com"]);
16542        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16543        run(&["add", "-A"]);
16544        run(&["commit", "-m", "init"]);
16545    }
16546
16547    // `settle_questions` is what closes the ghost the phone showed: a run's
16548    // seat asked something, the run then ended, and nothing was left to
16549    // abandon the question it left `open`. `HOME` is a process-wide
16550    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
16551    // first time it runs in the binary — every test below still reaches the
16552    // same directory whichever call won, and each gets its own run id from
16553    // `RunState::new`, so they never collide there.
16554    fn ask_test_home() {
16555        crate::run::pin_test_home();
16556    }
16557
16558    #[test]
16559    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16560        ask_test_home();
16561        let mut runner = runner_at(RunStatus::Prep);
16562        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16563        // A blocked temporary output path makes the terminal save fail while
16564        // preserving the already published reservation.
16565        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16566        let error = guard
16567            .finish(
16568                &mut runner.state,
16569                Err(
16570                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16571                        .into(),
16572                ),
16573            )
16574            .unwrap_err();
16575        assert_eq!(
16576            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16577            std::io::ErrorKind::PermissionDenied
16578        );
16579        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16580        let message = format!("{error:#}");
16581        assert!(message.contains("original refusal"));
16582        assert!(!message.contains("could not save terminal"));
16583        assert_eq!(
16584            error.chain().count(),
16585            2,
16586            "only startup attribution wraps the original error"
16587        );
16588        let saved = RunState::load(&runner.state.id).unwrap();
16589        assert_eq!(saved.status, RunStatus::Blocked);
16590        assert!(!saved.startup.unwrap().complete);
16591    }
16592
16593    #[test]
16594    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16595        ask_test_home();
16596        let mut runner = runner_at(RunStatus::Prep);
16597        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16598        let id = runner.state.id.clone();
16599        drop(guard);
16600        let saved = RunState::load(&id).unwrap();
16601        assert_eq!(saved.status, RunStatus::Blocked);
16602        assert!(!saved.startup.as_ref().unwrap().complete);
16603        assert!(
16604            saved
16605                .startup
16606                .unwrap()
16607                .failure
16608                .unwrap()
16609                .contains("interrupted")
16610        );
16611        assert!(Runner::resume(&id).is_err());
16612    }
16613
16614    #[test]
16615    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16616        ask_test_home();
16617        let mut runner = runner_at(RunStatus::Prep);
16618        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16619        let error = guard
16620            .finish(
16621                &mut runner.state,
16622                Err(
16623                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16624                        .into(),
16625                ),
16626            )
16627            .unwrap_err();
16628        assert_eq!(
16629            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16630            std::io::ErrorKind::PermissionDenied
16631        );
16632        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16633        let mut task = crate::queue::Task::new(
16634            "review".into(),
16635            "criteria".into(),
16636            runner.state.repo.clone(),
16637            crate::queue::Source::Human,
16638        );
16639        task.record_startup_failure(failure.clone());
16640        task.record_startup_failure(failure.clone());
16641        assert_eq!(task.startup_failures.len(), 1);
16642        assert!(task.runs.is_empty());
16643        assert_eq!(task.attempts, 0);
16644    }
16645
16646    #[cfg(unix)]
16647    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16648        ask_test_home();
16649        let tmp = tempfile::tempdir().unwrap();
16650        let repo = tmp.path().join("repo");
16651        std::fs::create_dir(&repo).unwrap();
16652        init_repo(&repo);
16653        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16654        let origin = tmp.path().join("origin.git");
16655        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16656            .await
16657            .unwrap();
16658        git::git(
16659            &repo,
16660            &["remote", "add", "origin", origin.to_str().unwrap()],
16661        )
16662        .await
16663        .unwrap();
16664        git::git(&repo, &["checkout", "-b", "fix/retained"])
16665            .await
16666            .unwrap();
16667        std::fs::write(repo.join("retained"), "retained change").unwrap();
16668        git::git(&repo, &["add", "."]).await.unwrap();
16669        git::git(&repo, &["commit", "-m", "retained implementation"])
16670            .await
16671            .unwrap();
16672        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16673        git::git(&repo, &["checkout", "main"]).await.unwrap();
16674        std::fs::write(repo.join("base-change"), "new base").unwrap();
16675        git::git(&repo, &["add", "."]).await.unwrap();
16676        git::git(&repo, &["commit", "-m", "base moved"])
16677            .await
16678            .unwrap();
16679        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16680        git::git(&repo, &["checkout", "fix/retained"])
16681            .await
16682            .unwrap();
16683        let mut runner = runner_at(RunStatus::Voting);
16684        runner.state.repo = repo.clone();
16685        runner.state.base_commit = base;
16686        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16687        runner.state.config.disk.min_free_bytes = 0;
16688        runner.state.config.graph.review_rounds = 1;
16689        runner.state.config.verify.e2e = vec!["true".into()];
16690        runner.state.config.verify.gate = vec!["true".into()];
16691        runner.state.candidates = vec![Candidate {
16692            index: 0,
16693            label: 'A',
16694            agent: "fixture".into(),
16695            branch: "fix/retained".into(),
16696            worktree: repo.clone(),
16697            summary: String::new(),
16698            stat: String::new(),
16699            files: 1,
16700            commits: 1,
16701            empty: false,
16702            failed: None,
16703            verified_noop: None,
16704            duration_ms: 0,
16705            folded: false,
16706        }];
16707        runner.state.tally = Some(Tally {
16708            first_choice: BTreeMap::new(),
16709            borda: BTreeMap::new(),
16710            winner: 'A',
16711            rankings: 0,
16712            unanimous_initial: true,
16713            deliberated: false,
16714            changed_votes: 0,
16715            unanimous_final: true,
16716            tie_break: None,
16717            judges: 0,
16718            present: 0,
16719            quorum: 0,
16720            met_quorum: true,
16721            uncontested: Some("review-only".into()),
16722        });
16723        let mut task = crate::queue::Task::new(
16724            "fixture".into(),
16725            "fixture".into(),
16726            repo,
16727            crate::queue::Source::Human,
16728        );
16729        task.link_run(&runner.state.id);
16730        crate::queue::Queue::open().put(&mut task).unwrap();
16731        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16732        runner.state.base_sync = Some(BaseSync {
16733            tip: String::new(),
16734            behind: 1,
16735            attempts: BASE_SYNC_ROUNDS,
16736            conflict: None,
16737            already_in: None,
16738        });
16739        runner.sync_to_base().await.unwrap();
16740        assert_eq!(runner.state.status, RunStatus::Blocked);
16741        (tmp, runner, head, task.id)
16742    }
16743
16744    #[cfg(unix)]
16745    #[tokio::test]
16746    async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
16747        for routed in [false, true] {
16748            let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
16749            let home = crate::run::home();
16750            let queue = crate::queue::Queue::open();
16751            let task = queue.get(&owner).unwrap();
16752            runner.state.task_instruction = Some(task.instruction.clone());
16753            runner.state.driver_exited = true;
16754            runner.state.save().unwrap();
16755            let questions = crate::ask::Questions::at(home.join("questions"));
16756            let mut q = crate::ask::Question::new(
16757                runner.state.id.clone(),
16758                "base-recovery".into(),
16759                "owner".into(),
16760                "Recover bounded base movement?".into(),
16761                String::new(),
16762                vec!["recover".into()],
16763            );
16764            q.actions.insert(
16765                "recover".into(),
16766                crate::ask::ChoiceAction::RecoverBase {
16767                    run: runner.state.id.clone(),
16768                    stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
16769                    episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
16770                },
16771            );
16772            q.routed_task = Some(owner.clone());
16773            q.answer(crate::ask::Answer::Choice("recover".into()))
16774                .unwrap();
16775            questions.put(&mut q).unwrap();
16776            crate::base_recovery_control::recover(
16777                &queue,
16778                &home,
16779                crate::base_recovery_control::Request {
16780                    run: runner.state.id.clone(),
16781                    head: head.clone(),
16782                    actor: "decisions".into(),
16783                    request: q.id.clone(),
16784                    decision: Some((owner.clone(), q.answered_at.unwrap())),
16785                    expected_stop: Some((
16786                        runner.state.base_sync_recovery.stop.clone().unwrap(),
16787                        runner.state.base_sync_recovery.episodes.last().unwrap().id,
16788                    )),
16789                },
16790            )
16791            .await
16792            .unwrap();
16793            if routed {
16794                questions
16795                    .update(&q.id, |q| {
16796                        q.routed_task = Some("another-owner".into());
16797                        Ok(())
16798                    })
16799                    .unwrap();
16800            } else {
16801                let mut task = queue.get(&owner).unwrap();
16802                task.hold_manual(Some("new owner decision".into()));
16803                // Keep the generation to exercise the separate owner-decision binding.
16804                task.status = crate::queue::TaskStatus::Queued;
16805                queue.put(&mut task).unwrap();
16806            }
16807            runner.state = RunState::load(&runner.state.id).unwrap();
16808            runner.sync_to_base().await.unwrap();
16809            let approval = runner
16810                .state
16811                .base_sync_recovery
16812                .authorizations
16813                .last()
16814                .unwrap();
16815            assert!(approval.consumed);
16816            assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
16817                "routing"
16818            } else {
16819                "owner decision"
16820            }));
16821            assert_eq!(
16822                runner.state.base_sync.as_ref().unwrap().attempts,
16823                BASE_SYNC_ROUNDS
16824            );
16825            assert_eq!(
16826                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16827                head
16828            );
16829        }
16830    }
16831
16832    #[cfg(unix)]
16833    #[tokio::test]
16834    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16835        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16836        if std::env::var_os(CHILD).is_none() {
16837            let home = tempfile::tempdir().unwrap();
16838            let output = std::process::Command::new(std::env::current_exe().unwrap())
16839                .args([
16840                    "--exact",
16841                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16842                    "--nocapture",
16843                ])
16844                .env(CHILD, "1")
16845                .env("MAGI_HOME", home.path())
16846                .env("MAGI_WORKER_HOME", home.path())
16847                .env("MAGI_WORKER_ID", "abcdef")
16848                .output()
16849                .unwrap();
16850            assert!(
16851                output.status.success()
16852                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16853                "{}\n{}",
16854                String::from_utf8_lossy(&output.stdout),
16855                String::from_utf8_lossy(&output.stderr)
16856            );
16857            return;
16858        }
16859        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16860        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16861        let home = crate::run::home();
16862        let queue = crate::queue::Queue::open();
16863        let mut owner = queue.get(&task).unwrap();
16864        owner.release_operator("test-release", None);
16865        owner.start(runner.state.id.clone());
16866        queue.put(&mut owner).unwrap();
16867        runner.state.driver_exited = true;
16868        runner.state.task_instruction = Some(owner.instruction.clone());
16869        runner.state.save().unwrap();
16870        let _claim = queue.claim(&task).unwrap();
16871        let reserve = |id: &str| {
16872            let directory = home.join("workers/jobs").join(id);
16873            std::fs::create_dir_all(&directory).unwrap();
16874            let job = crate::worker::Job {
16875                id: id.into(),
16876                sequence: 1,
16877                kind: "task".into(),
16878                key: format!("task:{task}"),
16879                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16880                generation: "fixture-generation".into(),
16881                version: None,
16882                status: crate::worker::Status::Running,
16883                result: None,
16884                pid: Some(std::process::id()),
16885                created_at: 0,
16886                finished_at: None,
16887            };
16888            std::fs::write(
16889                directory.join("job.json"),
16890                serde_json::to_vec(&job).unwrap(),
16891            )
16892            .unwrap();
16893            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16894        };
16895        let _own = reserve("abcdef");
16896        let other = reserve("fedcba");
16897        assert!(
16898            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16899                .await
16900                .is_err()
16901        );
16902        assert!(
16903            RunState::load(&runner.state.id)
16904                .unwrap()
16905                .base_sync_recovery
16906                .authorizations
16907                .is_empty()
16908        );
16909        drop(other);
16910        assert!(crate::daemon::is_working_on(
16911            &home,
16912            &runner.state.id,
16913            Timestamp::now()
16914        ));
16915        assert!(
16916            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16917        );
16918        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16919            .await
16920            .unwrap();
16921        assert!(
16922            RunState::load(&runner.state.id)
16923                .unwrap()
16924                .base_recovery_pending()
16925        );
16926    }
16927
16928    #[cfg(unix)]
16929    #[tokio::test]
16930    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16931        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16932        let home = crate::run::home();
16933        runner.state.driver_pid = Some(std::process::id());
16934        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16935        runner.state.driver_exited = false;
16936        runner.state.begin_pass();
16937        runner.state.save().unwrap();
16938        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16939        driver.record(&runner.state).unwrap();
16940        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16941        // Preserve the durable claim as if the process died before Drop saved.
16942        drop(driver);
16943        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16944        assert_eq!(
16945            runner.state.liveness_under(&home, false),
16946            crate::run::Liveness::Dead
16947        );
16948        let queue = crate::queue::Queue::open();
16949        let _claim = queue.claim(&task).unwrap();
16950        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16951            .await
16952            .unwrap();
16953        let disk = RunState::load(&runner.state.id).unwrap();
16954        assert!(!disk.driver_exited);
16955        assert!(disk.base_recovery_pending());
16956    }
16957
16958    #[cfg(unix)]
16959    #[tokio::test]
16960    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16961        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16962        if std::env::var_os(CHILD).is_none() {
16963            use std::os::unix::fs::PermissionsExt;
16964            let tmp = tempfile::tempdir().unwrap();
16965            let gh = tmp.path().join("gh");
16966            std::fs::write(&gh, r#"#!/bin/sh
16967case "$1 $2" in
16968  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16969  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16970  *) exit 2 ;;
16971esac
16972"#).unwrap();
16973            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16974            let mut paths = vec![tmp.path().to_path_buf()];
16975            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16976            let output = std::process::Command::new(std::env::current_exe().unwrap())
16977                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16978                .env(CHILD, "1")
16979                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16980                .env("PATH", std::env::join_paths(paths).unwrap())
16981                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16982                .output().unwrap();
16983            assert!(
16984                output.status.success()
16985                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16986                "{}\n{}",
16987                String::from_utf8_lossy(&output.stdout),
16988                String::from_utf8_lossy(&output.stderr)
16989            );
16990            return;
16991        }
16992        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16993        let repo = runner.state.repo.clone();
16994        git::git(&repo, &["checkout", "main"]).await.unwrap();
16995        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16996        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16997        git::git(&repo, &["checkout", "fix/retained"])
16998            .await
16999            .unwrap();
17000        runner.state.pr = Some(crate::run::PrRecord {
17001            url: "https://example.test/project/pull/1".into(),
17002            number: 1,
17003            state: "open".into(),
17004            checks: "unknown".into(),
17005            round: 0,
17006            rounds: 1,
17007            red_at_merge: Vec::new(),
17008        });
17009        runner.sync_to_base().await.unwrap();
17010        assert_eq!(runner.state.status, RunStatus::Blocked);
17011        assert_eq!(
17012            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17013            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17014        );
17015        assert!(runner.state.base_sync_closure_pending());
17016        assert!(crate::handover::can_resume(&runner.state));
17017        let queue = crate::queue::Queue::open();
17018        assert!(!crate::base_sync_recovery::held_task(
17019            &queue,
17020            &queue.get(&task).unwrap()
17021        ));
17022        assert!(runner.state.authorize_base_sync(&task).is_err());
17023        // Legacy closure stops have no typed stop but must retain the checkpoint.
17024        runner.state.base_sync_recovery.stop = None;
17025        runner.state.status = RunStatus::Voting;
17026        runner.state.gate_ran = true;
17027        runner.state.config.graph.review_rounds = 0;
17028        runner.state.save().unwrap();
17029        assert!(crate::handover::can_resume(&runner.state));
17030        runner.state = RunState::load(&runner.state.id).unwrap();
17031        runner.execute_graph().await.unwrap();
17032        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17033        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17034        assert_eq!(
17035            runner.state.base_sync.as_ref().unwrap().attempts,
17036            BASE_SYNC_ROUNDS
17037        );
17038        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17039        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17040        assert!(runner.state.reviews.is_empty());
17041        assert!(runner.state.gate.is_empty());
17042    }
17043
17044    #[cfg(unix)]
17045    #[tokio::test]
17046    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17047        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17048        runner.state.driver_exited = true;
17049        runner.state.save().unwrap();
17050        let home = crate::run::home();
17051        let queue = crate::queue::Queue::open();
17052        let _claim = queue.claim(&task).unwrap();
17053        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17054        assert!(
17055            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17056                .await
17057                .is_err()
17058        );
17059        drop(lease);
17060        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17061        assert!(
17062            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17063                .await
17064                .is_err()
17065        );
17066        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17067        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17068            .await
17069            .unwrap();
17070        let disk = RunState::load(&runner.state.id).unwrap();
17071        assert!(disk.base_recovery_pending());
17072        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17073        assert!(
17074            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17075                .await
17076                .is_err()
17077        );
17078        assert_eq!(
17079            RunState::load(&runner.state.id)
17080                .unwrap()
17081                .base_sync_recovery
17082                .authorizations
17083                .len(),
17084            1
17085        );
17086        let mut owner = queue.get(&task).unwrap();
17087        owner.runs.push("20260101-000000-ffff".into());
17088        queue.put(&mut owner).unwrap();
17089        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17090            .await
17091            .unwrap_err();
17092        assert!(error.to_string().contains("run task ownership changed"));
17093    }
17094
17095    #[cfg(unix)]
17096    #[tokio::test]
17097    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17098        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17099        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17100        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17101        legacy["status"] = serde_json::json!("voting");
17102        runner.state = serde_json::from_value(legacy).unwrap();
17103        runner.state.save().unwrap();
17104        for _ in 0..3 {
17105            runner.execute_graph().await.unwrap();
17106            assert_eq!(runner.state.status, RunStatus::Blocked);
17107            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17108            assert!(!crate::handover::can_resume(&runner.state));
17109            assert_eq!(
17110                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17111                head
17112            );
17113            assert!(runner.state.reviews.is_empty());
17114            assert!(!runner.state.gate_ran);
17115        }
17116        let notice = crate::notices::run_ended(&runner.state).unwrap();
17117        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17118        runner.state.begin_pass();
17119        runner.state.status = RunStatus::Voting;
17120        runner.deliberate().await.unwrap();
17121        runner.vote().await.unwrap();
17122        runner.finish_after_tally().await.unwrap();
17123        assert_eq!(runner.state.status, RunStatus::Blocked);
17124        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17125        assert_eq!(
17126            crate::notices::run_ended(&runner.state).unwrap().since,
17127            notice.since
17128        );
17129    }
17130
17131    #[cfg(unix)]
17132    #[tokio::test]
17133    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17134        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17135        for detail in [
17136            "fix/retained conflicts with origin/main and did not rebase: conflict",
17137            "rebased fix/retained locally but origin refused the push",
17138            "unknown failure",
17139        ] {
17140            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17141            runner.state.record_base_sync_stop(&head);
17142            assert!(runner.state.authorize_base_sync(&task).is_err());
17143            runner.state.status = RunStatus::Voting;
17144            runner.execute_graph().await.unwrap();
17145            assert_eq!(runner.state.status, RunStatus::Blocked);
17146            assert!(!crate::handover::can_resume(&runner.state));
17147            assert_eq!(
17148                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17149                head
17150            );
17151        }
17152    }
17153
17154    #[cfg(unix)]
17155    #[tokio::test]
17156    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17157        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17158        runner.state.authorize_base_sync(&task).unwrap();
17159        assert!(runner.state.consume_base_recovery(&head));
17160        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17161        runner.state.reserve_base_operation(&head, &tip);
17162        runner.state.save().unwrap();
17163        runner.state = RunState::load(&runner.state.id).unwrap();
17164        assert!(!runner.state.consume_base_recovery(&head));
17165        runner.sync_to_base().await.unwrap();
17166        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17167        assert_eq!(runner.state.base_episode_used(), 1);
17168        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17169        assert!(
17170            runner
17171                .state
17172                .base_sync_recovery
17173                .operations
17174                .last()
17175                .unwrap()
17176                .finished
17177        );
17178        assert!(!runner.state.base_sync_blocked());
17179        runner.sync_to_base().await.unwrap();
17180        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17181    }
17182
17183    #[cfg(unix)]
17184    #[tokio::test]
17185    async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17186        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17187        // Both historical diagnostic spellings retain an exhaustion identity.
17188        let original = runner
17189            .state
17190            .base_sync
17191            .as_ref()
17192            .unwrap()
17193            .conflict
17194            .clone()
17195            .unwrap();
17196        runner.state.base_sync.as_mut().unwrap().conflict = Some(
17197            original
17198                .replace("synchronization(s)", "rebase(s)")
17199                .replace("synchronizing", "rebasing"),
17200        );
17201        runner.state.base_sync_recovery = Default::default();
17202        runner.state.record_base_sync_stop(&head);
17203        assert_eq!(
17204            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17205            crate::base_sync_recovery::StopKind::Exhausted
17206        );
17207        runner.state.authorize_base_sync(&task).unwrap();
17208        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17209        let auth = legacy["base_sync_recovery"]["authorizations"][0]
17210            .as_object_mut()
17211            .unwrap();
17212        auth.remove("interval");
17213        auth.remove("scope");
17214        runner.state = serde_json::from_value(legacy).unwrap();
17215        assert!(runner.state.consume_base_recovery(&head));
17216        assert_eq!(runner.state.base_episode_used(), 3);
17217        runner.sync_to_base().await.unwrap();
17218        assert_eq!(runner.state.base_episode_used(), 4);
17219        runner.state.resource_block = Some(crate::run::ResourceBlock {
17220            node: "gate".into(),
17221            kind: "cache".into(),
17222            detail: "legacy wait".into(),
17223            at: Timestamp::now(),
17224            cause: crate::run::ResourceBlockCause::Unrecorded,
17225        });
17226        runner.state.save().unwrap();
17227        assert!(
17228            runner
17229                .state
17230                .base_sync_recovery
17231                .episodes
17232                .last()
17233                .unwrap()
17234                .boundary
17235                .is_none()
17236        );
17237        runner.state.resource_block = None;
17238        advance_base_fixture(&runner).await;
17239        runner.sync_to_base().await.unwrap();
17240        assert!(runner.state.base_sync_blocked());
17241        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17242    }
17243
17244    #[cfg(unix)]
17245    async fn advance_base_fixture(runner: &Runner) {
17246        let repo = &runner.state.repo;
17247        for args in [
17248            vec!["checkout", "main"],
17249            vec!["commit", "--allow-empty", "-m", "base advanced"],
17250            vec!["push", "origin", "main"],
17251            vec!["checkout", "fix/retained"],
17252        ] {
17253            git::git(repo, &args).await.unwrap();
17254        }
17255    }
17256
17257    #[cfg(unix)]
17258    fn fresh_base_interval_fixture(runner: &mut Runner) {
17259        runner.state.base_sync.as_mut().unwrap().attempts = 0;
17260        runner.state.base_sync.as_mut().unwrap().conflict = None;
17261        runner.state.base_sync_recovery = Default::default();
17262        runner.state.status = RunStatus::Voting;
17263    }
17264
17265    #[cfg(unix)]
17266    #[tokio::test]
17267    async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17268        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17269        fresh_base_interval_fixture(&mut runner);
17270        runner.state.begin_pass();
17271        runner.sync_to_base().await.unwrap();
17272        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17273        runner.state.resource_block = Some(crate::run::ResourceBlock {
17274            node: "gate".into(),
17275            kind: "build cache".into(),
17276            detail: "busy".into(),
17277            at: Timestamp::now(),
17278            cause: crate::run::ResourceBlockCause::Timeout {
17279                waited_secs: 1,
17280                limit_secs: 1,
17281            },
17282        });
17283        runner.state.save().unwrap();
17284        let queue = crate::queue::Queue::open();
17285        let mut owner = queue.get(&task).unwrap();
17286        runner.state.task_instruction = Some(owner.instruction.clone());
17287        owner.release_operator("fixture-owner", None);
17288        queue.put(&mut owner).unwrap();
17289        let approval = owner.resumption.clone().unwrap();
17290        runner.state.resource_block = None;
17291        runner.state.save().unwrap();
17292        runner.state = RunState::load(&runner.state.id).unwrap();
17293        advance_base_fixture(&runner).await;
17294        runner.sync_to_base().await.unwrap();
17295        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17296        assert_eq!(runner.state.base_episode_used(), 1);
17297        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17298        assert!(
17299            runner.state.base_sync_recovery.episodes[0]
17300                .boundary
17301                .as_ref()
17302                .unwrap()
17303                .consumed
17304        );
17305        assert!(
17306            !runner
17307                .state
17308                .renew_base_episode(&task, &approval, &head, "base")
17309        );
17310        for _ in 0..3 {
17311            owner.release_operator("fixture-owner", None);
17312            queue.put(&mut owner).unwrap();
17313            runner.state = RunState::load(&runner.state.id).unwrap();
17314            runner.state.begin_pass();
17315            runner.sync_to_base().await.unwrap();
17316            assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17317            assert_eq!(runner.state.base_episode_used(), 1);
17318        }
17319        runner.state.resource_block = Some(crate::run::ResourceBlock {
17320            node: "gate".into(),
17321            kind: "disk pressure".into(),
17322            detail: "free space required".into(),
17323            at: Timestamp::now(),
17324            cause: crate::run::ResourceBlockCause::Other,
17325        });
17326        runner.state.end_pass_parked(ParkCause::Interrupt {
17327            reason: "disk pressure".into(),
17328        });
17329        runner.state.save().unwrap();
17330        owner.release_operator("fixture-owner-after-disk", None);
17331        queue.put(&mut owner).unwrap();
17332        runner.state.resource_block = None;
17333        advance_base_fixture(&runner).await;
17334        runner.sync_to_base().await.unwrap();
17335        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17336        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17337        assert_eq!(runner.state.base_episode_used(), 1);
17338        assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17339    }
17340
17341    #[cfg(unix)]
17342    #[tokio::test]
17343    async fn base_recovery_continuous_advancement_stops_at_four() {
17344        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17345        fresh_base_interval_fixture(&mut runner);
17346        for spent in 1..=BASE_SYNC_ROUNDS {
17347            runner.sync_to_base().await.unwrap();
17348            assert!(!runner.state.base_sync_blocked());
17349            assert_eq!(runner.state.base_episode_used(), spent);
17350            advance_base_fixture(&runner).await;
17351        }
17352        runner.sync_to_base().await.unwrap();
17353        assert_eq!(
17354            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17355            crate::base_sync_recovery::StopKind::Exhausted
17356        );
17357        let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17358        assert!(stop.unverified_sync);
17359        let action = runner.state.base_sync_next_action();
17360        assert!(action.contains("never reached its gate"));
17361        assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17362        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17363        runner.state = RunState::load(&runner.state.id).unwrap();
17364        runner.sync_to_base().await.unwrap();
17365        assert_eq!(runner.state.base_episode_used(), 4);
17366        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17367    }
17368
17369    #[cfg(unix)]
17370    #[tokio::test]
17371    async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17372        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17373        runner.state.ensure_base_episode();
17374        // Verification progress alone is not evidence of a successful sync.
17375        runner.state.record_base_progress(&head);
17376        runner.state.base_sync_recovery.stop = None;
17377        runner.state.record_base_sync_stop(&head);
17378        assert!(
17379            !runner
17380                .state
17381                .base_sync_recovery
17382                .stop
17383                .as_ref()
17384                .unwrap()
17385                .unverified_sync
17386        );
17387
17388        runner.state.finish_base_operation(Some(&head));
17389        // A gate for another head cannot verify this synchronization.
17390        runner.state.gate_ran = true;
17391        runner.state.gate_head = Some("older-head".into());
17392        runner.state.base_sync_recovery.stop = None;
17393        runner.state.record_base_sync_stop(&head);
17394        runner.state.save().unwrap();
17395        runner.state = RunState::load(&runner.state.id).unwrap();
17396        assert!(
17397            runner
17398                .state
17399                .base_sync_recovery
17400                .stop
17401                .as_ref()
17402                .unwrap()
17403                .unverified_sync
17404        );
17405
17406        runner.state.gate_head = Some(head.clone());
17407        runner.state.base_sync_recovery.stop = None;
17408        runner.state.record_base_sync_stop(&head);
17409        assert!(
17410            !runner
17411                .state
17412                .base_sync_recovery
17413                .stop
17414                .as_ref()
17415                .unwrap()
17416                .unverified_sync
17417        );
17418        assert!(
17419            !runner
17420                .state
17421                .base_sync_next_action()
17422                .contains("never reached its gate")
17423        );
17424
17425        // Historical records cannot establish the missing synchronization evidence.
17426        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17427        legacy["base_sync_recovery"]["stop"]
17428            .as_object_mut()
17429            .unwrap()
17430            .remove("unverified_sync");
17431        legacy["base_sync_recovery"]["episodes"][0]
17432            .as_object_mut()
17433            .unwrap()
17434            .remove("last_sync_head");
17435        runner.state = serde_json::from_value(legacy).unwrap();
17436        assert!(
17437            !runner
17438                .state
17439                .base_sync_recovery
17440                .stop
17441                .as_ref()
17442                .unwrap()
17443                .unverified_sync
17444        );
17445        assert!(
17446            runner.state.base_sync_recovery.episodes[0]
17447                .last_sync_head
17448                .is_none()
17449        );
17450    }
17451
17452    #[cfg(unix)]
17453    #[tokio::test]
17454    async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17455        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17456        runner.state.authorize_base_sync(&task).unwrap();
17457        assert!(
17458            runner
17459                .state
17460                .base_sync_next_action()
17461                .contains("Approval recorded")
17462        );
17463        assert!(
17464            runner
17465                .state
17466                .base_sync_next_action()
17467                .contains("4/4 in episode 0, 4 lifetime")
17468        );
17469        assert!(runner.state.consume_base_recovery(&head));
17470        assert!(
17471            runner
17472                .state
17473                .base_sync_next_action()
17474                .contains("0/4 in episode 1, 4 lifetime")
17475        );
17476    }
17477
17478    #[cfg(unix)]
17479    #[tokio::test]
17480    async fn base_recovery_changed_head_invalidates_without_spending() {
17481        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17482        runner.state.authorize_base_sync(&task).unwrap();
17483        git::git(
17484            &runner.state.repo,
17485            &["commit", "--allow-empty", "-m", "external head change"],
17486        )
17487        .await
17488        .unwrap();
17489        runner.sync_to_base().await.unwrap();
17490        let a = runner
17491            .state
17492            .base_sync_recovery
17493            .authorizations
17494            .last()
17495            .unwrap();
17496        assert!(a.consumed);
17497        assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17498        runner.state = RunState::load(&runner.state.id).unwrap();
17499        assert!(
17500            runner
17501                .state
17502                .base_sync_next_action()
17503                .contains("Approval invalidated")
17504        );
17505        assert!(!runner.state.base_recovery_pending());
17506        assert!(runner.state.base_sync_blocked());
17507        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17508    }
17509
17510    #[cfg(unix)]
17511    #[tokio::test]
17512    async fn base_recovery_stale_fetched_base_invalidates_approval() {
17513        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17514        runner.state.driver_exited = true;
17515        runner.state.save().unwrap();
17516        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17517            .await
17518            .unwrap();
17519        runner.state = RunState::load(&runner.state.id).unwrap();
17520        advance_base_fixture(&runner).await;
17521        runner.sync_to_base().await.unwrap();
17522        let a = runner
17523            .state
17524            .base_sync_recovery
17525            .authorizations
17526            .last()
17527            .unwrap();
17528        assert!(
17529            a.consumed
17530                && a.invalidated
17531                    .as_deref()
17532                    .unwrap()
17533                    .contains("fetched base changed")
17534        );
17535        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17536        assert_eq!(
17537            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17538            head
17539        );
17540        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17541            .await
17542            .unwrap();
17543        runner.state = RunState::load(&runner.state.id).unwrap();
17544        runner.sync_to_base().await.unwrap();
17545        assert!(!runner.state.base_sync_blocked());
17546        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17547    }
17548
17549    #[cfg(unix)]
17550    #[tokio::test]
17551    async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17552        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17553        runner.state.authorize_base_sync(&task).unwrap();
17554        assert!(runner.state.consume_base_recovery(&head));
17555        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17556        runner.state.reserve_base_operation(&head, &tip);
17557        runner.state.save().unwrap();
17558        let scratch = runner.state.dir().join("base-sync");
17559        assert_eq!(
17560            crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17561                .await
17562                .unwrap(),
17563            crate::rebase::Rebased::Applied
17564        );
17565        runner.state = RunState::load(&runner.state.id).unwrap();
17566        runner.sync_to_base().await.unwrap();
17567        assert!(
17568            runner
17569                .state
17570                .base_sync_recovery
17571                .operations
17572                .last()
17573                .unwrap()
17574                .finished
17575        );
17576        assert_eq!(runner.state.base_episode_used(), 1);
17577        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17578        assert!(!runner.state.base_sync_blocked());
17579    }
17580
17581    #[cfg(unix)]
17582    #[tokio::test]
17583    async fn base_recovery_crash_preserves_untracked_files() {
17584        for ignored in [false, true] {
17585            let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17586            runner.state.authorize_base_sync(&task).unwrap();
17587            assert!(runner.state.consume_base_recovery(&head));
17588            let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17589            runner.state.reserve_base_operation(&head, &tip);
17590            runner.state.save().unwrap();
17591            let scratch = runner.state.dir().join("base-sync");
17592            assert_eq!(
17593                crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17594                    .await
17595                    .unwrap(),
17596                crate::rebase::Rebased::Applied
17597            );
17598            let repo = &runner.state.repo;
17599            let result = git::rev_parse(repo, "HEAD").await.unwrap();
17600            assert_ne!(result, head);
17601            if ignored {
17602                let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17603                    .await
17604                    .unwrap();
17605                std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17606            }
17607            std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17608            assert!(
17609                git::git_raw(repo, &["diff", "--quiet", &head])
17610                    .await
17611                    .unwrap()
17612                    .ok()
17613            );
17614            // Resume after the branch moved, before its checkout was reset.
17615            runner.state = RunState::load(&runner.state.id).unwrap();
17616            runner.sync_to_base().await.unwrap();
17617            assert!(runner.state.base_sync_blocked());
17618            assert!(!runner.state.base_sync_recovery.operations[0].finished);
17619            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17620            assert_eq!(
17621                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17622                result
17623            );
17624            assert_eq!(
17625                std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17626                "retained work\n"
17627            );
17628            assert!(
17629                git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17630                    .await
17631                    .unwrap()
17632                    .ok()
17633            );
17634        }
17635    }
17636
17637    #[cfg(unix)]
17638    #[tokio::test]
17639    async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17640        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17641        runner.state.authorize_base_sync(&task).unwrap();
17642        assert!(runner.state.consume_base_recovery(&head));
17643        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17644        runner.state.reserve_base_operation(&head, &tip);
17645        runner.state.save().unwrap();
17646        let scratch = runner.state.dir().join("base-sync");
17647        crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17648            .await
17649            .unwrap();
17650        git::sync_to_head(&runner.state.repo).await.unwrap();
17651        std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17652        git::git(&runner.state.repo, &["add", "foreign"])
17653            .await
17654            .unwrap();
17655        git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17656            .await
17657            .unwrap();
17658        let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17659        runner.state = RunState::load(&runner.state.id).unwrap();
17660        runner.sync_to_base().await.unwrap();
17661        assert!(runner.state.base_sync_blocked());
17662        assert!(
17663            !runner
17664                .state
17665                .base_sync_recovery
17666                .operations
17667                .last()
17668                .unwrap()
17669                .finished
17670        );
17671        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17672        assert_eq!(
17673            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17674            foreign_head
17675        );
17676    }
17677
17678    #[cfg(unix)]
17679    async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17680    {
17681        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17682        fresh_base_interval_fixture(&mut runner);
17683        let repo = &runner.state.repo;
17684        git::git(repo, &["checkout", "main"]).await.unwrap();
17685        std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17686        git::git(repo, &["add", "retained"]).await.unwrap();
17687        git::git(repo, &["commit", "-m", "upstream retained change"])
17688            .await
17689            .unwrap();
17690        git::git(repo, &["push", "origin", "main"]).await.unwrap();
17691        let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17692        git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17693        let scratch = runner.state.dir().join("base-sync");
17694        assert!(matches!(
17695            git::rebase_start(repo, &scratch, "fix/retained", &tip)
17696                .await
17697                .unwrap(),
17698            git::RebaseStart::Conflicted(_)
17699        ));
17700        runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17701            agent: "fixture".into(),
17702            paths: vec!["retained".into()],
17703            from: Some(head.clone()),
17704            stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17705            index_outside: None,
17706            finished: false,
17707            error: None,
17708            launcher: None,
17709            infra: None,
17710            outage: None,
17711        });
17712        (tmp, runner, head, tip, scratch)
17713    }
17714
17715    #[cfg(unix)]
17716    #[tokio::test]
17717    async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17718        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17719        // A resource stop from before reservations existed, with resolved
17720        // contents left for the launcher's ordinary staging continuation.
17721        std::fs::write(
17722            scratch.join("retained"),
17723            "upstream and retained resolution\n",
17724        )
17725        .unwrap();
17726        runner.state.resource_block = Some(crate::run::ResourceBlock {
17727            node: "rebase".into(),
17728            kind: "repository coordination".into(),
17729            detail: "busy".into(),
17730            at: Timestamp::now(),
17731            cause: crate::run::ResourceBlockCause::Other,
17732        });
17733        runner.state.save().unwrap();
17734        // Advance the remote without moving either checkout participating
17735        // in the standing rebase.
17736        let newer = git::git(
17737            &runner.state.repo,
17738            &[
17739                "commit-tree",
17740                "main^{tree}",
17741                "-p",
17742                "main",
17743                "-m",
17744                "base advanced during wait",
17745            ],
17746        )
17747        .await
17748        .unwrap();
17749        git::git(
17750            &runner.state.repo,
17751            &[
17752                "push",
17753                "origin",
17754                &format!("{}:refs/heads/main", newer.trim()),
17755            ],
17756        )
17757        .await
17758        .unwrap();
17759        runner.state = RunState::load(&runner.state.id).unwrap();
17760        runner.state.resource_block = None;
17761        runner.sync_to_base().await.unwrap();
17762        assert!(!runner.state.base_sync_blocked());
17763        let op = &runner.state.base_sync_recovery.operations[0];
17764        assert_eq!(op.head, head);
17765        assert_eq!(op.tip, tip);
17766        assert!(op.finished);
17767        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17768        assert_eq!(runner.state.rebase_fixes.len(), 1);
17769        let latest = git::rev_parse(&runner.state.repo, "origin/main")
17770            .await
17771            .unwrap();
17772        assert_ne!(latest, tip);
17773        runner.state = RunState::load(&runner.state.id).unwrap();
17774        runner.sync_to_base().await.unwrap();
17775        assert!(!runner.state.base_sync_blocked());
17776        assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17777        assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17778        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17779    }
17780
17781    #[cfg(unix)]
17782    #[tokio::test]
17783    async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17784        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17785        runner.state.reserve_base_operation(&head, &tip);
17786        runner.state.save().unwrap();
17787        std::fs::write(
17788            scratch.join("retained"),
17789            "upstream and retained resolution\n",
17790        )
17791        .unwrap();
17792        git::git(&scratch, &["add", "retained"]).await.unwrap();
17793        git::git(
17794            &scratch,
17795            &["-c", "core.editor=true", "rebase", "--continue"],
17796        )
17797        .await
17798        .unwrap();
17799        let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17800        assert!(
17801            !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17802                .await
17803                .unwrap()
17804                .is_empty()
17805        );
17806        assert_eq!(
17807            crate::rebase::finish(
17808                &mut runner.state,
17809                &scratch,
17810                "fix/retained",
17811                &head,
17812                &tip,
17813                &["retained".into()],
17814                ""
17815            )
17816            .await
17817            .unwrap(),
17818            crate::rebase::Rebased::Applied
17819        );
17820        assert!(!scratch.exists());
17821        // Simulate losing the caller before finish_base_operation/save.
17822        runner.state = RunState::load(&runner.state.id).unwrap();
17823        let op = &runner.state.base_sync_recovery.operations[0];
17824        assert!(!op.finished);
17825        assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17826        runner.sync_to_base().await.unwrap();
17827        assert!(!runner.state.base_sync_blocked());
17828        assert!(runner.state.base_sync_recovery.operations[0].finished);
17829        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17830        assert_eq!(runner.state.rebase_fixes.len(), 1);
17831        let winner = runner.state.winner().unwrap().clone();
17832        let wait = runner.repository_wait();
17833        assert!(
17834            verify_current_head(&mut runner.state, &winner, &wait)
17835                .await
17836                .unwrap()
17837        );
17838        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17839    }
17840
17841    #[cfg(unix)]
17842    #[tokio::test]
17843    async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17844        for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17845            let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17846            runner.state.reserve_base_operation(&head, &tip);
17847            runner.state.save().unwrap();
17848            std::fs::write(
17849                scratch.join("retained"),
17850                "upstream and retained resolution\n",
17851            )
17852            .unwrap();
17853            git::git(&scratch, &["add", "retained"]).await.unwrap();
17854            git::git(
17855                &scratch,
17856                &["-c", "core.editor=true", "rebase", "--continue"],
17857            )
17858            .await
17859            .unwrap();
17860            let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17861            assert!(!git::rebase_in_progress(&scratch).await);
17862            assert!(
17863                !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17864                    .await
17865                    .unwrap()
17866                    .is_empty()
17867            );
17868            match damage {
17869                Some("record") => {
17870                    let path =
17871                        git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17872                            .await
17873                            .unwrap();
17874                    std::fs::write(scratch.join(path), "").unwrap();
17875                }
17876                Some("foreign") => {
17877                    std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17878                    git::git(&scratch, &["add", "foreign"]).await.unwrap();
17879                    git::git(&scratch, &["commit", "-m", "foreign work"])
17880                        .await
17881                        .unwrap();
17882                }
17883                Some("dirty") => {
17884                    std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17885                }
17886                _ => {}
17887            }
17888            let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17889            // Crash immediately after Git continuation, before finish runs.
17890            runner.state = RunState::load(&runner.state.id).unwrap();
17891            assert!(
17892                runner.state.base_sync_recovery.operations[0]
17893                    .validated_head
17894                    .is_none()
17895            );
17896            runner.sync_to_base().await.unwrap();
17897            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17898            assert_eq!(runner.state.base_episode_used(), 1);
17899            assert_eq!(runner.state.rebase_fixes.len(), 1);
17900            assert_eq!(
17901                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17902                retained_head
17903            );
17904            if damage.is_some() {
17905                assert!(runner.state.base_sync_blocked());
17906                assert!(scratch.exists());
17907                assert!(!runner.state.base_sync_recovery.operations[0].finished);
17908                assert!(
17909                    runner.state.base_sync_recovery.operations[0]
17910                        .validated_head
17911                        .is_none()
17912                );
17913            } else {
17914                assert!(!runner.state.base_sync_blocked());
17915                assert!(!scratch.exists());
17916                assert!(runner.state.base_sync_recovery.operations[0].finished);
17917                assert_eq!(
17918                    runner.state.base_sync_recovery.operations[0]
17919                        .validated_head
17920                        .as_deref(),
17921                    Some(result.as_str())
17922                );
17923                let winner = runner.state.winner().unwrap().clone();
17924                let wait = runner.repository_wait();
17925                assert!(
17926                    verify_current_head(&mut runner.state, &winner, &wait)
17927                        .await
17928                        .unwrap()
17929                );
17930                assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17931                runner.state = RunState::load(&runner.state.id).unwrap();
17932                runner.sync_to_base().await.unwrap();
17933                assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17934            }
17935        }
17936    }
17937
17938    #[cfg(unix)]
17939    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17940        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17941        // This is an ordinary gate stop, not an authorized base-budget recovery.
17942        runner.state.prep_complete = Some(true);
17943        runner.state.base_sync = None;
17944        runner.state.base_sync_recovery = Default::default();
17945        runner.state.config.graph.gate_fix_rounds = 1;
17946        let mut review = review_round(true, 0, 1, 1, false, true);
17947        review.head = head.clone();
17948        review.verified_head = Some(head.clone());
17949        runner.state.reviews = vec![review];
17950        runner.state.gate = vec![CommandOutcome {
17951            command: "old gate".into(),
17952            code: Some(1),
17953            output_tail: "old failure".into(),
17954            duration_ms: 0,
17955            resource_blocked: false,
17956        }];
17957        runner.state.gate_head = Some(head.clone());
17958        runner.state.gate_ran = true;
17959        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17960            agent: "fixture".into(),
17961            failed: runner.state.gate.clone(),
17962            notes: "spent".into(),
17963            committed: false,
17964            error: None,
17965        }];
17966        runner.state.land_approval = Some(crate::run::LandApproval {
17967            question: "old approval".into(),
17968            head: head.clone(),
17969        });
17970        runner.state.status = RunStatus::Blocked;
17971        (tmp, runner, head)
17972    }
17973
17974    #[cfg(unix)]
17975    #[tokio::test]
17976    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17977        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17978            let (tmp, mut runner, old) = resumed_gate_fixture().await;
17979            let phases = tmp.path().join("phases");
17980            runner.state.config.verify.e2e = vec![format!(
17981                "echo e2e >> '{}'; {}",
17982                phases.display(),
17983                if e2e_ok {
17984                    "true"
17985                } else {
17986                    "echo new-e2e-failure; false"
17987                }
17988            )];
17989            runner.state.config.verify.gate = vec![format!(
17990                "echo gate >> '{}'; {}",
17991                phases.display(),
17992                if gate_ok {
17993                    "true"
17994                } else {
17995                    "echo new-gate-failure; false"
17996                }
17997            )];
17998            runner.state.save().unwrap();
17999            runner.state = RunState::load(&runner.state.id).unwrap();
18000            runner.sync_to_base().await.unwrap();
18001            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18002            assert_ne!(head, old);
18003            // Reentry after Git updated but before verification was invalidated.
18004            runner.state = RunState::load(&runner.state.id).unwrap();
18005            runner.execute_graph().await.unwrap();
18006            assert_eq!(runner.state.gate_fixes.len(), 1);
18007            assert_eq!(runner.state.reviews.len(), 1);
18008            assert_eq!(runner.state.reviews[0].head, old);
18009            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18010            assert_eq!(runner.state.verification_history.len(), 1);
18011            let history = &runner.state.verification_history[0];
18012            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18013            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18014            assert!(history.archived_at.is_some());
18015            assert_eq!(history.fixes_spent, Some(1));
18016            assert_eq!(history.gate[0].output_tail, "old failure");
18017            let archive_events: Vec<_> = runner
18018                .state
18019                .events
18020                .iter()
18021                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18022                .collect();
18023            assert_eq!(archive_events.len(), 1);
18024            assert!(archive_events[0].message.contains(&short(&old)));
18025            assert!(archive_events[0].message.contains(&short(&head)));
18026            assert!(
18027                archive_events[0]
18028                    .message
18029                    .contains("1 gate-fix round(s) already spent stay spent")
18030            );
18031            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18032            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18033            if e2e_ok {
18034                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18035                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18036            } else {
18037                assert!(!runner.state.gate_ran);
18038            }
18039            assert_eq!(
18040                runner.state.status,
18041                if e2e_ok && gate_ok {
18042                    RunStatus::Ready
18043                } else {
18044                    RunStatus::Blocked
18045                }
18046            );
18047            if !e2e_ok || !gate_ok {
18048                let cause = runner.state.verification_blocking_cause().unwrap();
18049                assert!(cause.contains(&head));
18050                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18051                // A later successful progress event must not become the cause.
18052                runner.state.event("land", "rebased successfully");
18053                let notice = crate::notices::run_ended(&runner.state).unwrap();
18054                assert!(!format!("{notice:?}").contains("rebased successfully"));
18055                assert!(runner.state.merge.is_none());
18056            }
18057            let before = std::fs::read_to_string(&phases).unwrap();
18058            runner.state.save().unwrap();
18059            runner.state = RunState::load(&runner.state.id).unwrap();
18060            runner.review_loop().await.unwrap();
18061            runner.gate().await.unwrap();
18062            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18063            assert_eq!(runner.state.verification_history.len(), 1);
18064            let history = &runner.state.verification_history[0];
18065            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18066            assert!(history.archived_at.is_some());
18067            assert_eq!(history.fixes_spent, Some(1));
18068        }
18069    }
18070
18071    #[cfg(unix)]
18072    #[tokio::test]
18073    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18074        let (tmp, mut runner, head) = resumed_gate_fixture().await;
18075        let marker = tmp.path().join("gate-ran");
18076        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18077        runner.review_loop().await.unwrap();
18078        runner.gate().await.unwrap();
18079        assert_eq!(runner.state.status, RunStatus::Blocked);
18080        assert!(!marker.exists());
18081        assert!(runner.state.verification_history.is_empty());
18082        runner.state.gate_head = None;
18083        runner.gate().await.unwrap();
18084        assert!(marker.exists());
18085        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18086        assert_eq!(runner.state.verification_history.len(), 1);
18087        assert_eq!(runner.state.gate_fixes.len(), 1);
18088    }
18089
18090    #[cfg(unix)]
18091    #[tokio::test]
18092    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18093        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18094        runner.sync_to_base().await.unwrap();
18095        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18096        runner.state.reconcile_verification_head(&head);
18097        // Both crash-before-command and resource wait are unresolved evidence.
18098        for outcomes in [
18099            Vec::new(),
18100            vec![CommandOutcome {
18101                command: "true".into(),
18102                code: None,
18103                output_tail: "cache unavailable".into(),
18104                duration_ms: 0,
18105                resource_blocked: true,
18106            }],
18107        ] {
18108            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18109                head: head.clone(),
18110                passed: false,
18111                outcomes,
18112            });
18113            runner.state.gate_ran = false;
18114            runner.state.gate.clear();
18115            runner.state.save().unwrap();
18116            runner.state = RunState::load(&runner.state.id).unwrap();
18117            runner.merge().await.unwrap();
18118            assert!(runner.state.merge.is_none());
18119            runner.review_loop().await.unwrap();
18120            runner.gate().await.unwrap();
18121            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18122            assert!(runner.state.gate_allows_progress());
18123            assert_eq!(runner.state.gate_fixes.len(), 1);
18124            assert!(runner.state.merge.is_none());
18125        }
18126        // Moving the winner after a successful gate cannot publish an unchecked SHA.
18127        git::git(
18128            &runner.state.repo,
18129            &["commit", "--allow-empty", "-m", "external update"],
18130        )
18131        .await
18132        .unwrap();
18133        runner.merge().await.unwrap();
18134        assert!(runner.state.merge.is_none());
18135        assert_eq!(runner.state.status, RunStatus::Blocked);
18136    }
18137
18138    #[cfg(unix)]
18139    #[tokio::test]
18140    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18141        for waiting_on_gate in [false, true] {
18142            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18143            runner.sync_to_base().await.unwrap();
18144            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18145            runner.state.reconcile_verification_head(&head);
18146            if waiting_on_gate {
18147                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18148                    head: head.clone(),
18149                    passed: true,
18150                    outcomes: Vec::new(),
18151                });
18152            }
18153            let cache = tmp.path().join("cache");
18154            runner.state.config.verify.e2e =
18155                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18156            runner.state.config.verify.gate = vec!["true".into()];
18157            runner.state.config.graph.timeout_resource_wait = 1;
18158            let owner =
18159                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18160            let guard =
18161                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18162                    crate::cache::AcquireOutcome::Acquired(g) => g,
18163                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18164                };
18165            runner.review_loop().await.unwrap();
18166            runner.gate().await.unwrap();
18167            assert!(!runner.state.gate_ran);
18168            assert_eq!(runner.state.gate_fixes.len(), 1);
18169            assert!(runner.state.resource_block.is_some());
18170            runner.merge().await.unwrap();
18171            assert!(runner.state.merge.is_none());
18172            drop(guard);
18173            runner.state = RunState::load(&runner.state.id).unwrap();
18174            // The driver clears the previous pass's resource stop on resume.
18175            runner.state.resource_block = None;
18176            runner.review_loop().await.unwrap();
18177            runner.gate().await.unwrap();
18178            assert!(runner.state.gate_allows_progress());
18179            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18180            assert_eq!(runner.state.gate_fixes.len(), 1);
18181        }
18182    }
18183
18184    #[cfg(unix)]
18185    #[tokio::test]
18186    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18187        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18188        runner.sync_to_base().await.unwrap();
18189        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18190        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18191        // Verification passes before pre_gate and fails on the mechanically changed head.
18192        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18193        runner.review_loop().await.unwrap();
18194        runner.gate().await.unwrap();
18195        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18196        assert_ne!(before, after);
18197        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18198        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18199        assert!(!runner.state.gate_allows_progress());
18200        runner.merge().await.unwrap();
18201        assert!(runner.state.merge.is_none());
18202        assert!(
18203            runner
18204                .state
18205                .verification_history
18206                .iter()
18207                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18208        );
18209
18210        // An incomplete review is still an independent hold on a changed head.
18211        runner.state.reviews[0].clean = false;
18212        runner.state.reviews[0].answered = 0;
18213        runner.state.config.verify.e2e = vec!["true".into()];
18214        runner.review_loop().await.unwrap();
18215        runner.gate().await.unwrap();
18216        runner.merge().await.unwrap();
18217        assert_eq!(runner.state.status, RunStatus::Blocked);
18218        assert!(runner.state.merge.is_none());
18219        assert!(
18220            runner
18221                .state
18222                .verification_blocking_cause()
18223                .unwrap()
18224                .contains("review blocked")
18225        );
18226    }
18227
18228    #[cfg(unix)]
18229    #[tokio::test]
18230    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18231        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18232        runner.state.gate_head = None;
18233        runner.state.config.verify.gate =
18234            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18235        runner.review_loop().await.unwrap();
18236        runner.gate().await.unwrap();
18237        assert_ne!(
18238            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18239            old
18240        );
18241        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18242        assert!(!runner.state.gate_allows_progress());
18243        assert_eq!(runner.state.status, RunStatus::Blocked);
18244        runner.merge().await.unwrap();
18245        assert!(runner.state.merge.is_none());
18246    }
18247
18248    #[cfg(unix)]
18249    #[tokio::test]
18250    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18251        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18252        let mut old_review = review_round(true, 0, 1, 1, false, true);
18253        old_review.head = head.clone();
18254        old_review.verified_head = Some(head.clone());
18255        runner.state.reviews = vec![old_review];
18256        runner.state.gate = vec![CommandOutcome {
18257            command: "old gate".into(),
18258            code: None,
18259            output_tail: "timed out after 2400s".into(),
18260            duration_ms: 2_400_000,
18261            resource_blocked: false,
18262        }];
18263        runner.state.gate_ran = true;
18264        runner.state.gate_head = Some(head.clone());
18265        runner.state.land_approval = Some(crate::run::LandApproval {
18266            question: "old-head-approval".into(),
18267            head: head.clone(),
18268        });
18269        runner.state.authorize_base_sync(&task).unwrap();
18270        runner.sync_to_base().await.unwrap();
18271        assert!(!runner.state.base_sync_blocked());
18272        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18273        assert_ne!(head, fresh);
18274        assert_eq!(runner.state.reviews.len(), 1);
18275        assert!(!runner.state.gate_ran);
18276        let saved = &runner.state.base_sync_recovery.authorizations[0];
18277        assert_eq!(saved.reviews[0].head, head);
18278        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18279        assert!(saved.gate_ran);
18280        assert!(!saved.gate[0].ok());
18281        runner.review_loop().await.unwrap();
18282        runner.gate().await.unwrap();
18283        assert_eq!(runner.state.reviews[0].head, head);
18284        assert_eq!(
18285            runner.state.reviews[0].verified_head.as_deref(),
18286            Some(head.as_str())
18287        );
18288        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18289        assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18290        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18291        assert!(runner.state.gate_ran);
18292        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18293        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18294        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18295        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18296        assert_eq!(
18297            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18298            "retained change"
18299        );
18300    }
18301
18302    #[cfg(unix)]
18303    #[tokio::test]
18304    async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18305        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18306        let queue = crate::queue::Queue::open();
18307        let mut owner = queue.get(&task).unwrap();
18308        owner.release_operator("test-resume", None);
18309        queue.put(&mut owner).unwrap();
18310        let generation = owner.resumption.as_ref().unwrap().generation;
18311        runner.state.task_instruction = Some(owner.instruction.clone());
18312        runner.state.driver_exited = true;
18313        runner.state.save().unwrap();
18314        let home = crate::run::home();
18315        let _claim = queue.claim(&task).unwrap();
18316        owner.resumption.as_mut().unwrap().at =
18317            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18318        queue.put(&mut owner).unwrap();
18319        assert!(
18320            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18321                .await
18322                .is_err(),
18323            "an approval from before the stop grants nothing"
18324        );
18325        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18326        queue.put(&mut owner).unwrap();
18327        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18328            .await
18329            .unwrap();
18330        assert!(
18331            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18332                .await
18333                .is_err()
18334        );
18335        runner.state = RunState::load(&runner.state.id).unwrap();
18336        assert_eq!(
18337            runner.state.base_sync_recovery.authorizations[0].generation,
18338            Some(generation)
18339        );
18340        runner.sync_to_base().await.unwrap();
18341        let repo = runner.state.repo.clone();
18342        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18343        git::git(&repo, &["checkout", "main"]).await.unwrap();
18344        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18345        git::git(&repo, &["add", "."]).await.unwrap();
18346        git::git(&repo, &["commit", "-m", "move base again"])
18347            .await
18348            .unwrap();
18349        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18350        git::git(&repo, &["checkout", "fix/retained"])
18351            .await
18352            .unwrap();
18353        runner.sync_to_base().await.unwrap();
18354        assert!(!runner.state.base_sync_blocked());
18355        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18356        assert_eq!(runner.state.base_episode_used(), 2);
18357        let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18358        assert_ne!(fresh, head);
18359        assert!(!runner.state.base_recovery_pending());
18360        // Even replaying the generation with a later timestamp grants nothing.
18361        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18362        queue.put(&mut owner).unwrap();
18363        assert!(
18364            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18365                .await
18366                .is_err()
18367        );
18368        runner.state = RunState::load(&runner.state.id).unwrap();
18369        runner.sync_to_base().await.unwrap();
18370        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18371        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18372        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18373    }
18374
18375    #[cfg(unix)]
18376    #[tokio::test]
18377    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18378        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18379        runner.state.base_sync = None;
18380        runner.state.status = RunStatus::Blocked;
18381        let mut second = runner.state.candidates[0].clone();
18382        second.label = 'B';
18383        runner.state.candidates.push(second);
18384        assert!(runner.state.votes.is_empty());
18385        assert!(runner.state.deliberation.is_empty());
18386        runner.deliberate().await.unwrap();
18387        runner.vote().await.unwrap();
18388        assert_eq!(runner.state.status, RunStatus::Blocked);
18389        assert!(runner.state.votes.is_empty());
18390        assert!(runner.state.deliberation.is_empty());
18391    }
18392
18393    /// A minimal, git-free `Runner` at a given status — `settle_questions`
18394    /// reads nothing else off it.
18395    fn runner_at(status: RunStatus) -> Runner {
18396        let mut state = RunState::new(
18397            PathBuf::from("/nonexistent/repo"),
18398            "main".to_owned(),
18399            "deadbeef".to_owned(),
18400            "task".to_owned(),
18401            Config::default(),
18402        );
18403        // These fixtures test execution flow, independent of host disk capacity.
18404        state.config.disk.min_free_bytes = 0;
18405        // These fixtures share one home and save concurrently. Short clock
18406        // IDs can collide, making an ownership test read another test's run.
18407        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18408        state.id = format!(
18409            "20000101-000000-{:04x}",
18410            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18411        );
18412        state.status = status;
18413        Runner {
18414            state,
18415            roles: ResolvedRoles {
18416                implementers: Vec::new(),
18417                judges: Vec::new(),
18418                reviewers: Vec::new(),
18419                fixer: None,
18420                conductor: conductor(),
18421                implementer_roster: Vec::new(),
18422                judge_roster: Vec::new(),
18423                reviewer_roster: Vec::new(),
18424            },
18425            sem: Arc::new(Semaphore::new(1)),
18426            pause: Pause::new(),
18427            interrupt: Pause::new(),
18428        }
18429    }
18430
18431    #[tokio::test]
18432    async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18433        crate::run::pin_test_home();
18434        let temp = tempfile::tempdir().unwrap();
18435        let root = temp.path().canonicalize().unwrap();
18436        let repo = root.join("repo");
18437        std::fs::create_dir(&repo).unwrap();
18438        init_repo(&repo);
18439        git::git(&repo, &["config", "core.autocrlf", "false"])
18440            .await
18441            .unwrap();
18442        let mut runner = runner_at(RunStatus::Blocked);
18443        runner.state.repo = repo.clone();
18444        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18445        runner.state.config.agents = vec![conductor()];
18446        runner.state.config.graph.selected_implementer = Some("conductor".into());
18447        runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18448        let (shell, command, setup_output) = if cfg!(windows) {
18449            (
18450                vec!["cmd".into(), "/C".into()],
18451                "echo ready>seat-ready",
18452                "ready\r\n",
18453            )
18454        } else {
18455            (
18456                vec!["sh".into(), "-c".into()],
18457                "printf ready > seat-ready",
18458                "ready",
18459            )
18460        };
18461        runner.state.config.verify.shell = Some(shell);
18462        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18463            run: Some(command.into()),
18464            ..Default::default()
18465        }];
18466        runner.state.save().unwrap();
18467        let path = runner.state.worktree_root().join("judge-1");
18468        git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18469            .await
18470            .unwrap();
18471        std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18472        git::git(&path, &["add", "."]).await.unwrap();
18473        let tree = git::git(&path, &["write-tree"]).await.unwrap();
18474        let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18475            .await
18476            .unwrap();
18477        git::git(&path, &["checkout", "--detach", &head])
18478            .await
18479            .unwrap();
18480        let evidence = crate::fold_evidence::snapshot(
18481            &runner.state,
18482            &crate::run::home().canonicalize().unwrap(),
18483            &head,
18484            64 * 1024 * 1024,
18485        )
18486        .await
18487        .unwrap();
18488        let mut contents = BTreeMap::new();
18489        contents.insert(
18490            repo.join("logs/HEAD"),
18491            format!("{head} {head} fixture\n").into_bytes(),
18492        );
18493        let record = serde_json::json!({
18494            "version":1, "run":runner.state.id, "source":path, "head":head,
18495            "evidence":evidence, "admin":{}, "admin_contents":contents,
18496            "history":{head.clone():evidence}, "phase":"evicted"
18497        });
18498        std::fs::write(
18499            runner.state.dir().join("artifacts/eviction-judge-1.json"),
18500            serde_json::to_vec(&record).unwrap(),
18501        )
18502        .unwrap();
18503        assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18504        git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18505            .await
18506            .unwrap();
18507        git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18508        assert!(
18509            !git::git_raw(&repo, &["cat-file", "-e", &head])
18510                .await
18511                .unwrap()
18512                .ok()
18513        );
18514        let mut resumed = Runner::resume(&runner.state.id).unwrap();
18515        let error = resumed.execute().await.unwrap_err();
18516        assert!(
18517            error
18518                .to_string()
18519                .contains("selected implementation is held"),
18520            "{error:#}"
18521        );
18522        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18523        assert_eq!(
18524            std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18525            setup_output
18526        );
18527        assert_eq!(
18528            std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18529            "unrelated seat"
18530        );
18531        let error = resumed.execute().await.unwrap_err();
18532        assert!(
18533            error
18534                .to_string()
18535                .contains("selected implementation is held"),
18536            "{error:#}"
18537        );
18538        std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18539        git::git(&path, &["add", "."]).await.unwrap();
18540        let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18541        let next = git::git(
18542            &path,
18543            &[
18544                "commit-tree",
18545                &next_tree,
18546                "-p",
18547                &head,
18548                "-m",
18549                "next review checkpoint",
18550            ],
18551        )
18552        .await
18553        .unwrap();
18554        git::git(&path, &["checkout", "--detach", &next])
18555            .await
18556            .unwrap();
18557        resumed.state.reviews.push(
18558            serde_json::from_value(serde_json::json!({
18559                "round":1, "head":next, "reviews":[]
18560            }))
18561            .unwrap(),
18562        );
18563        resumed.state.save().unwrap();
18564        let error = resumed.execute().await.unwrap_err();
18565        assert!(
18566            error
18567                .to_string()
18568                .contains("selected implementation is held"),
18569            "{error:#}"
18570        );
18571        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18572    }
18573
18574    #[tokio::test]
18575    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18576        crate::run::pin_test_home();
18577        let mut runner = runner_at(RunStatus::Gating);
18578        runner.state.save().unwrap();
18579        let id = runner.state.id.clone();
18580        let passes = runner.state.passes.len();
18581        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18582        let mut claiming = Box::pin(runner.claim_driver());
18583        std::future::poll_fn(|cx| {
18584            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18585            std::task::Poll::Ready(())
18586        })
18587        .await;
18588        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18589        drop(owner);
18590        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18591            .await
18592            .unwrap()
18593            .unwrap();
18594        assert_eq!(execution.runner.state.passes.len(), passes + 1);
18595        assert!(execution.runner.state.resource_block.is_none());
18596        assert_eq!(
18597            execution.runner.state.liveness(false),
18598            crate::run::Liveness::Live
18599        );
18600        drop(execution);
18601        assert!(RunState::load(&id).unwrap().driver_exited);
18602    }
18603
18604    #[tokio::test]
18605    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18606        crate::run::pin_test_home();
18607        let mut runner = runner_at(RunStatus::Gating);
18608        runner.state.save().unwrap();
18609        // run.json can commit, but the subsequent lease metadata write fails.
18610        // Cleanup must run after releasing the short ownership lock.
18611        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18612        assert!(runner.claim_driver().await.is_err());
18613        assert!(runner.state.driver_exited);
18614        assert_eq!(runner.state.driver_pid, None);
18615        let disk = RunState::load(&runner.state.id).unwrap();
18616        assert!(disk.driver_exited);
18617        assert_eq!(disk.driver_pid, None);
18618        assert_eq!(disk.status, RunStatus::Gating);
18619    }
18620
18621    #[tokio::test]
18622    async fn an_early_graph_error_releases_its_driver() {
18623        crate::run::pin_test_home();
18624        let mut runner = runner_at(RunStatus::Blocked);
18625        runner.state.config.graph.selected_implementer = Some("selected".into());
18626        runner.state.save().unwrap();
18627        assert!(runner.execute().await.is_err());
18628        let disk = RunState::load(&runner.state.id).unwrap();
18629        assert!(disk.driver_exited);
18630        assert_eq!(disk.driver_pid, None);
18631        assert_eq!(disk.status, RunStatus::Blocked);
18632    }
18633
18634    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18635        crate::run::MergeRepair {
18636            question: "repair-question".into(),
18637            head: "old-head".into(),
18638            pr: "https://github.com/o/r/pull/42".into(),
18639            reviews: vec![review_round(false, 1, 1, 1, false, true)],
18640            gate: vec![outcome(Some(0), false)],
18641            gate_head: Some("old-head".into()),
18642            gate_fixes: Vec::new(),
18643            pre_gate: Vec::new(),
18644            phase,
18645            failure: None,
18646        }
18647    }
18648
18649    #[tokio::test]
18650    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18651        crate::run::pin_test_home();
18652        let mut runner = runner_at(RunStatus::Landing);
18653        runner.state.merge_repairs.push(merge_repair_request(
18654            crate::run::MergeRepairPhase::Requested,
18655        ));
18656        runner.state.save().unwrap();
18657        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18658        assert!(
18659            runner
18660                .execute()
18661                .await
18662                .unwrap_err()
18663                .to_string()
18664                .contains("already running")
18665        );
18666        assert_eq!(
18667            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18668            crate::run::MergeRepairPhase::Requested
18669        );
18670        drop(claim);
18671        let mut disk = runner.state.clone();
18672        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18673        disk.save().unwrap();
18674        assert!(
18675            runner
18676                .execute()
18677                .await
18678                .unwrap_err()
18679                .to_string()
18680                .contains("decision changed")
18681        );
18682        assert_eq!(
18683            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18684            crate::run::MergeRepairPhase::Complete
18685        );
18686    }
18687
18688    #[test]
18689    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18690        crate::run::pin_test_home();
18691        let mut runner = runner_at(RunStatus::Landing);
18692        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18693        runner.state.reviews = request.reviews.clone();
18694        runner.state.gate = request.gate.clone();
18695        runner.state.gate_ran = true;
18696        runner.state.land_approval = Some(crate::run::LandApproval {
18697            question: "old".into(),
18698            head: "old-head".into(),
18699        });
18700        runner.state.merge_repairs.push(request);
18701        runner.begin_merge_repair_review(0).unwrap();
18702        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18703        assert!(!runner.state.gate_status().ok());
18704        assert!(runner.state.land_approval.is_none());
18705        let disk = RunState::load(&runner.state.id).unwrap();
18706        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18707        assert!(disk.merge_repairs[0].gate[0].ok());
18708        assert_eq!(
18709            disk.merge_repairs[0].phase,
18710            crate::run::MergeRepairPhase::Reviewing
18711        );
18712        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18713        assert!(disk.reviews.is_empty());
18714    }
18715
18716    #[tokio::test]
18717    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18718        crate::run::pin_test_home();
18719        for phase in [
18720            crate::run::MergeRepairPhase::Fixing,
18721            crate::run::MergeRepairPhase::Requested,
18722        ] {
18723            let mut runner = runner_at(RunStatus::Landing);
18724            runner.state.config.graph.land_rounds = 0;
18725            runner.state.merge_repairs.push(merge_repair_request(phase));
18726            runner.state.save().unwrap();
18727            assert!(!runner.repair_merge_review().await.unwrap());
18728            assert_eq!(runner.state.status, RunStatus::Blocked);
18729            assert_eq!(
18730                runner.state.merge_repairs[0].phase,
18731                crate::run::MergeRepairPhase::Failed
18732            );
18733            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18734            assert!(
18735                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18736                    "interrupted"
18737                } else {
18738                    "budget exhausted"
18739                })
18740            );
18741            assert!(runner.state.merge.is_none());
18742        }
18743    }
18744
18745    /// `park_here` folding in the reason `Pause::park_because` recorded -
18746    /// this is what lets an operator reading a run's events tell an
18747    /// interrupt-driven park from an ordinary shutdown park.
18748    #[test]
18749    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18750        crate::run::pin_test_home();
18751        let mut runner = runner_at(RunStatus::Implementing);
18752        let interrupt = Pause::new();
18753        runner.watch_interrupt(interrupt.clone());
18754
18755        interrupt.park_because("task a1b2 asked to run first");
18756
18757        assert!(runner.park_here().expect("park_here"));
18758        assert!(runner.state.parked);
18759        let last = runner.state.events.last().expect("a park event");
18760        assert_eq!(last.node, "park");
18761        assert!(
18762            last.message.contains("task a1b2 asked to run first"),
18763            "expected the interrupt reason in {:?}",
18764            last.message
18765        );
18766    }
18767
18768    /// The cause is stored on the pass when it parks, and the first cause a
18769    /// shared pause was given is the one that stays.
18770    #[test]
18771    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18772        crate::run::pin_test_home();
18773        let mut runner = runner_at(RunStatus::Implementing);
18774        runner.state.begin_pass();
18775        let shutdown = Pause::new();
18776        runner.on_pause(shutdown.clone());
18777        let upgrade = ParkCause::Upgrade {
18778            from: "0.121.0".to_owned(),
18779            to: Some("0.121.1".to_owned()),
18780        };
18781        shutdown.park_for(upgrade.clone());
18782        shutdown.park_for(ParkCause::Operator);
18783
18784        assert!(runner.park_here().expect("park_here"));
18785        assert_eq!(
18786            runner.state.passes[0].end,
18787            Some(crate::run::PassEnd::Parked { cause: upgrade })
18788        );
18789        let last = runner.state.events.last().expect("a park event");
18790        assert!(last.message.contains("0.121.0"), "{}", last.message);
18791    }
18792
18793    #[tokio::test]
18794    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18795        crate::run::pin_test_home();
18796        let mut runner = runner_at(RunStatus::Reviewing);
18797        runner.state.save().unwrap();
18798        let mut execution = runner.claim_driver().await.expect("first execution");
18799        assert_eq!(execution.runner.state.passes.len(), 1);
18800        execution
18801            .driver
18802            .finish(&mut execution.runner.state)
18803            .await
18804            .unwrap();
18805    }
18806
18807    #[tokio::test]
18808    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18809        crate::run::pin_test_home();
18810        let mut runner = runner_at(RunStatus::Reviewing);
18811        runner.state.config.agents = vec![conductor()];
18812        runner.state.config.graph.timeout_resource_wait = 0;
18813        runner.state.save().unwrap();
18814        let id = runner.state.id.clone();
18815        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18816        assert!(runner.claim_driver().await.is_err());
18817        drop(owner);
18818        drop(runner);
18819        let mut resumed = Runner::resume(&id).unwrap();
18820        assert_eq!(resumed.state.driver_claimed, Some(false));
18821        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18822        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18823        let mut execution = resumed
18824            .claim_driver()
18825            .await
18826            .expect("retry initial claim after reload");
18827        assert_eq!(execution.runner.state.driver_claimed, Some(true));
18828        assert_eq!(execution.runner.state.passes.len(), 1);
18829        execution
18830            .driver
18831            .finish(&mut execution.runner.state)
18832            .await
18833            .unwrap();
18834    }
18835
18836    #[tokio::test]
18837    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18838        crate::run::pin_test_home();
18839        let mut runner = runner_at(RunStatus::Reviewing);
18840        runner.state.config.agents = vec![conductor()];
18841        runner.state.save().unwrap();
18842        let id = runner.state.id.clone();
18843        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18844        let mut resumed = Runner::resume(&id).unwrap();
18845        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18846        assert!(resumed.claim_driver().await.is_err());
18847        assert!(RunState::load(&id).unwrap().passes.is_empty());
18848        drop(lease);
18849        let mut execution = resumed.claim_driver().await.unwrap();
18850        execution
18851            .driver
18852            .finish(&mut execution.runner.state)
18853            .await
18854            .unwrap();
18855    }
18856
18857    #[tokio::test]
18858    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18859        crate::run::pin_test_home();
18860        let mut runner = runner_at(RunStatus::Reviewing);
18861        let state = &mut runner.state;
18862        state.config.agents = vec![conductor()];
18863        state.driver_claimed = None;
18864        state.driver_pid = Some(std::process::id());
18865        state.driver_started_at = None;
18866        state.begin_pass();
18867        state.save().unwrap();
18868        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18869        std::fs::write(
18870            state.dir().join("driver.json"),
18871            serde_json::to_vec(&serde_json::json!({
18872                "pid": state.driver_pid, "started_at": null,
18873                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18874            }))
18875            .unwrap(),
18876        )
18877        .unwrap();
18878        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18879        drop(lease);
18880        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18881        let mut resumed = Runner::resume(&state.id).unwrap();
18882        assert!(!resumed.state.driver_exited);
18883        let mut execution = resumed
18884            .claim_driver()
18885            .await
18886            .expect("lease proves previous execution ended");
18887        assert_eq!(execution.runner.state.passes.len(), 2);
18888        execution
18889            .driver
18890            .finish(&mut execution.runner.state)
18891            .await
18892            .unwrap();
18893    }
18894
18895    #[tokio::test]
18896    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18897        crate::run::pin_test_home();
18898        let mut runner = runner_at(RunStatus::Reviewing);
18899        let mut json = serde_json::to_value(&runner.state).unwrap();
18900        json.as_object_mut().unwrap().remove("driver_claimed");
18901        runner.state = serde_json::from_value(json).unwrap();
18902        assert_eq!(runner.state.driver_claimed, None);
18903        runner.state.save().unwrap();
18904        let mut execution = runner
18905            .claim_driver()
18906            .await
18907            .expect("the execution lease, not a missing marker, excludes a second driver");
18908        assert_eq!(execution.runner.state.passes.len(), 1);
18909        execution
18910            .driver
18911            .finish(&mut execution.runner.state)
18912            .await
18913            .unwrap();
18914    }
18915
18916    #[tokio::test]
18917    async fn driver_admission_refuses_a_proven_live_driver() {
18918        crate::run::pin_test_home();
18919        let mut runner = runner_at(RunStatus::Reviewing);
18920        let pid = std::process::id();
18921        runner.state.driver_pid = Some(pid);
18922        runner.state.driver_started_at = crate::proc::process_started_at(pid);
18923        assert!(runner.state.driver_started_at.is_some());
18924        runner.state.save().unwrap();
18925        let error = runner
18926            .claim_driver()
18927            .await
18928            .err()
18929            .expect("a live driver must refuse");
18930        assert!(error.to_string().contains("live driver"), "{error:#}");
18931        let saved = RunState::load(&runner.state.id).unwrap();
18932        assert_eq!(saved.driver_pid, Some(pid));
18933        assert_eq!(saved.status, RunStatus::Reviewing);
18934        assert!(saved.passes.is_empty());
18935    }
18936
18937    #[test]
18938    fn parking_resource_halt_persists_the_pass_once() {
18939        crate::run::pin_test_home();
18940        let mut runner = runner_at(RunStatus::Reviewing);
18941        runner.state.begin_pass();
18942        runner.state.resource_block = Some(crate::run::ResourceBlock {
18943            node: "fix".into(),
18944            kind: "shared build cache".into(),
18945            detail: "cancelled".into(),
18946            at: Timestamp::now(),
18947            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18948        });
18949        let cause = ParkCause::Upgrade {
18950            from: "old".into(),
18951            to: Some("new".into()),
18952        };
18953        runner.pause.park_for(cause.clone());
18954        // Both direct fixer exits and graph resource boundaries must complete
18955        // parking, rather than returning a stale Reviewing record.
18956        runner.stop_for_resource().unwrap();
18957        assert!(runner.resource_halt().unwrap());
18958        assert!(runner.park_here().unwrap());
18959        let saved = RunState::load(&runner.state.id).unwrap();
18960        assert!(saved.parked);
18961        assert_eq!(saved.status, RunStatus::Reviewing);
18962        assert!(saved.passes[0].ended_at.is_some());
18963        assert_eq!(
18964            saved.passes[0].end,
18965            Some(crate::run::PassEnd::Parked {
18966                cause: cause.clone()
18967            })
18968        );
18969        assert_eq!(
18970            saved.resource_block.unwrap().cause,
18971            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18972        );
18973        assert_eq!(
18974            saved
18975                .events
18976                .iter()
18977                .filter(|event| event.node == "park")
18978                .count(),
18979            1
18980        );
18981    }
18982
18983    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
18984    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
18985    /// cleared) must not make a *different* run - one only watching its own,
18986    /// unshared interrupt `Pause` - see itself as parked. If a future change
18987    /// ever collapsed these back into one handle, the interrupt scheduler
18988    /// would park every run for the rest of the daemon's life, not just the
18989    /// one it meant to interrupt.
18990    #[test]
18991    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
18992        crate::run::pin_test_home();
18993        let mut runner = runner_at(RunStatus::Implementing);
18994        let shutdown = Pause::new();
18995        runner.on_pause(shutdown.clone());
18996        let interrupt = Pause::new();
18997        runner.watch_interrupt(interrupt.clone());
18998
18999        // Nobody has asked for anything yet.
19000        assert!(!runner.park_here().expect("park_here"));
19001        assert!(!runner.state.parked);
19002
19003        // Only the interrupt handle fires; the shutdown handle stays clear.
19004        interrupt.park_because("test");
19005        assert!(!shutdown.parked());
19006        assert!(runner.park_here().expect("park_here"));
19007    }
19008
19009    /// The property every prior attempt at this feature failed to pin down:
19010    /// asking a run to park while one of its nodes has a real, in-flight
19011    /// async operation running (an agent call, in production) must not cut
19012    /// that operation short. `park_here` is only ever consulted *between*
19013    /// `execute`'s node calls - see its own doc - so nothing inside a node
19014    /// can observe a park request until the node itself returns. This proves
19015    /// that structurally, with real `tokio` concurrency and a channel
19016    /// handshake (never a sleep, which would only prove "usually", not
19017    /// "cannot"): the "node" below reports that it has genuinely started,
19018    /// and only then is the park requested; the node still has to be told to
19019    /// finish before `park_here` is ever called, exactly mirroring every
19020    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
19021    /// in `execute`.
19022    #[tokio::test]
19023    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19024        crate::run::pin_test_home();
19025        let mut runner = runner_at(RunStatus::Implementing);
19026        let interrupt = Pause::new();
19027        runner.watch_interrupt(interrupt.clone());
19028
19029        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19030        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19031
19032        // Stands in for one node's in-flight agent call: it proves it has
19033        // genuinely started, then blocks - exactly as a spawned CLI process
19034        // does - until told to finish.
19035        let node = async move {
19036            started_tx.send(()).expect("send started");
19037            finish_rx.await.expect("recv finish");
19038            "node finished"
19039        };
19040
19041        let interrupter = async move {
19042            started_rx.await.expect("recv started");
19043            // The call is now genuinely in flight. Ask it to park.
19044            interrupt.park_because("higher-priority task waiting");
19045            // Nothing the node does can observe this yet - there is no
19046            // check inside it, by construction - so let the executor run
19047            // anything pending and then let the node finish on its own.
19048            tokio::task::yield_now().await;
19049            finish_tx.send(()).expect("send finish");
19050        };
19051
19052        let (node_result, ()) = tokio::join!(node, interrupter);
19053        assert_eq!(
19054            node_result, "node finished",
19055            "the in-flight call ran to completion"
19056        );
19057
19058        // Only now, at the boundary the real `execute` would check right
19059        // after this node, does the park take effect.
19060        assert!(runner.park_here().expect("park_here"));
19061        assert!(runner.state.parked);
19062    }
19063
19064    /// A run parked mid-competition carries every field it had accumulated
19065    /// through the exact same disk round-trip an ordinary resume uses -
19066    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
19067    /// Nothing about parking for an interrupt is a special case of that path;
19068    /// this is what proves it rather than assuming it.
19069    #[test]
19070    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19071        crate::run::pin_test_home();
19072        let mut runner = runner_at(RunStatus::Judging);
19073        // `Runner::resume` re-resolves roles from the saved config, which
19074        // refuses an empty roster - give it the same minimal one `conductor`
19075        // itself uses.
19076        runner.state.config.agents = vec![conductor()];
19077        runner.state.candidates = vec![Candidate {
19078            index: 0,
19079            label: 'A',
19080            agent: "alpha".to_owned(),
19081            branch: "magi/x/A".to_owned(),
19082            worktree: PathBuf::from("/nonexistent/worktree"),
19083            summary: "did the thing".to_owned(),
19084            stat: "1 file changed".to_owned(),
19085            files: 1,
19086            commits: 1,
19087            empty: false,
19088            failed: None,
19089            verified_noop: None,
19090            duration_ms: 1234,
19091            folded: false,
19092        }];
19093        let run_id = runner.state.id.clone();
19094
19095        let interrupt = Pause::new();
19096        runner.watch_interrupt(interrupt.clone());
19097        interrupt.park_because("task c3d4 asked to run first");
19098        assert!(runner.park_here().expect("park_here"));
19099
19100        let resumed = Runner::resume(&run_id).expect("resume");
19101        assert_eq!(resumed.state.candidates.len(), 1);
19102        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19103        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19104        assert_eq!(resumed.state.status, runner.state.status);
19105        assert!(
19106            resumed.state.parked,
19107            "still parked until `execute` actually walks the graph again"
19108        );
19109        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19110    }
19111
19112    /// A fresh open question on `run`, stored and handed back for assertions.
19113    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19114        let mut q = ask::Question::new(
19115            run.to_owned(),
19116            "implement".to_owned(),
19117            "impl-A".to_owned(),
19118            "Which storage backend should the cache use?".to_owned(),
19119            String::new(),
19120            vec!["SQLite".to_owned(), "Redis".to_owned()],
19121        );
19122        store.put(&mut q).unwrap();
19123        q
19124    }
19125
19126    #[test]
19127    fn a_failed_runs_open_question_is_abandoned() {
19128        ask_test_home();
19129        let store = ask::Questions::open();
19130        let mut runner = runner_at(RunStatus::Failed);
19131        let run = runner.state.id.clone();
19132        let q = ask_open_question(&store, &run);
19133
19134        runner.settle_questions();
19135
19136        let back = store.get(&q.id).unwrap();
19137        assert!(
19138            !back.status.open(),
19139            "the seat that asked died with the run; nobody is left to read an answer"
19140        );
19141        assert!(
19142            back.detail.contains(&run) && back.detail.contains("failed"),
19143            "the reason names what the run became, not just that it is gone: {}",
19144            back.detail
19145        );
19146    }
19147
19148    #[test]
19149    fn a_merged_runs_open_question_is_abandoned_too() {
19150        ask_test_home();
19151        let store = ask::Questions::open();
19152        // A run that finishes cleanly still leaves nobody to read an answer -
19153        // this is not only a failure-path cleanup.
19154        for status in [RunStatus::Merged, RunStatus::Ready] {
19155            let mut runner = runner_at(status);
19156            let run = runner.state.id.clone();
19157            let q = ask_open_question(&store, &run);
19158
19159            runner.settle_questions();
19160
19161            let back = store.get(&q.id).unwrap();
19162            assert!(
19163                !back.status.open(),
19164                "{status:?} run's question must not outlive the run"
19165            );
19166        }
19167    }
19168
19169    #[test]
19170    fn a_still_resumable_runs_open_question_is_left_alone() {
19171        ask_test_home();
19172        let store = ask::Questions::open();
19173        // `Blocked` and `Stalled` can still be resumed — the candidates, the
19174        // review round and the seat sessions are all still on disk — so a
19175        // question asked mid-round may yet get a real answer from a real
19176        // resume. Sweeping it here would be exactly the failure mode this
19177        // whole feature exists to avoid on the other side.
19178        for status in [RunStatus::Blocked, RunStatus::Stalled] {
19179            let mut runner = runner_at(status);
19180            let run = runner.state.id.clone();
19181            let q = ask_open_question(&store, &run);
19182
19183            runner.settle_questions();
19184
19185            let back = store.get(&q.id).unwrap();
19186            assert!(
19187                back.status.open(),
19188                "{status:?} is still alive; the question must still be waiting"
19189            );
19190        }
19191    }
19192
19193    #[test]
19194    fn settle_questions_never_touches_an_already_answered_question() {
19195        ask_test_home();
19196        let store = ask::Questions::open();
19197        let mut runner = runner_at(RunStatus::Failed);
19198        let run = runner.state.id.clone();
19199        let mut q = ask_open_question(&store, &run);
19200        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19201            .unwrap();
19202        store.put(&mut q).unwrap();
19203
19204        // Called twice, the way a crash-recovered daemon reclaim and the
19205        // graph's own cleanup both can for the same run — `abandon_for_run`
19206        // only ever touches what is still open, so this must be inert both
19207        // times, not merely the second.
19208        runner.settle_questions();
19209        runner.settle_questions();
19210
19211        let back = store.get(&q.id).unwrap();
19212        assert_eq!(
19213            back.status,
19214            ask::QuestionStatus::Answered,
19215            "a real answer is a decision on record, never overwritten by a sweep"
19216        );
19217    }
19218
19219    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
19220    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
19221    /// without merging, whose winner is still the operator's answer to read.
19222    /// Nothing previously called `fold_run` itself with a real `tally`, so
19223    /// this is the first test to pin down the one distinction the whole
19224    /// automatic-fold feature depends on: the winner's worktree and branch
19225    /// must survive, everything else sharing the run's worktree bay - a
19226    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
19227    /// second sweep treats every non-winner directory under the bay alike -
19228    /// must not.
19229    #[tokio::test]
19230    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19231        crate::run::pin_test_home();
19232        let tmp = tempfile::tempdir().expect("tempdir");
19233        let repo = tmp.path().join("repo");
19234        std::fs::create_dir_all(&repo).unwrap();
19235        init_repo(&repo);
19236
19237        let mut config = Config::default();
19238        config.graph.worktree_root = Some(tmp.path().join("wt"));
19239
19240        let mut state = RunState::new(
19241            repo.clone(),
19242            "main".to_owned(),
19243            "deadbeef".to_owned(),
19244            "task".to_owned(),
19245            config,
19246        );
19247        let root = state.worktree_root();
19248        let wt_a = root.join("cand-A");
19249        let wt_b = root.join("cand-B");
19250        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19251            .await
19252            .expect("worktree A");
19253        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19254            .await
19255            .expect("worktree B");
19256
19257        state.candidates = vec![
19258            Candidate {
19259                index: 0,
19260                label: 'A',
19261                agent: "alpha".to_owned(),
19262                branch: "magi/x/A".to_owned(),
19263                worktree: wt_a.clone(),
19264                summary: String::new(),
19265                stat: String::new(),
19266                files: 0,
19267                commits: 0,
19268                empty: false,
19269                failed: None,
19270                verified_noop: None,
19271                duration_ms: 0,
19272                folded: false,
19273            },
19274            Candidate {
19275                index: 1,
19276                label: 'B',
19277                agent: "beta".to_owned(),
19278                branch: "magi/x/B".to_owned(),
19279                worktree: wt_b.clone(),
19280                summary: String::new(),
19281                stat: String::new(),
19282                files: 0,
19283                commits: 0,
19284                empty: false,
19285                failed: None,
19286                verified_noop: None,
19287                duration_ms: 0,
19288                folded: false,
19289            },
19290        ];
19291        state.tally = Some(Tally {
19292            first_choice: BTreeMap::from([('A', 1)]),
19293            borda: BTreeMap::new(),
19294            winner: 'A',
19295            rankings: 1,
19296            unanimous_initial: true,
19297            deliberated: false,
19298            changed_votes: 0,
19299            unanimous_final: true,
19300            tie_break: None,
19301            judges: 1,
19302            present: 1,
19303            quorum: 1,
19304            met_quorum: true,
19305            uncontested: None,
19306        });
19307        state.status = RunStatus::Ready;
19308
19309        fold_run(&mut state, false, &crate::run::home())
19310            .await
19311            .expect("fold_run");
19312
19313        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19314        assert!(
19315            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19316            "the unmerged winner's branch survives"
19317        );
19318        assert!(
19319            !state.candidates[0].folded,
19320            "the winner is not marked folded"
19321        );
19322
19323        assert!(!wt_b.exists(), "the loser's worktree is removed");
19324        assert!(
19325            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19326            "the loser's branch is removed"
19327        );
19328        assert!(state.candidates[1].folded, "the loser is marked folded");
19329    }
19330
19331    #[tokio::test]
19332    #[cfg(unix)]
19333    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19334        use std::os::fd::AsRawFd as _;
19335        use std::time::Duration;
19336
19337        let tmp = tempfile::tempdir().unwrap();
19338        let repo = tmp.path().join("repo");
19339        std::fs::create_dir_all(&repo).unwrap();
19340        init_repo(&repo);
19341        let common = repo.join(".git");
19342        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19343        let home = tmp.path().join("home");
19344        let mut config = Config::default();
19345        config.coordination = crate::config::test_coordination();
19346        config.graph.worktree_root = Some(tmp.path().join("wt"));
19347        config.graph.timeout_resource_wait = 1;
19348        let mut state = RunState::new(
19349            repo.clone(),
19350            "main".into(),
19351            "deadbeef".into(),
19352            "task".into(),
19353            config,
19354        );
19355        let root = state.worktree_root();
19356        let candidate = root.join("cand-A");
19357        let reviewer = root.join("review-1");
19358        let branch = state.branch_for('A');
19359        git::worktree_add_branch(&repo, &candidate, &branch, "main")
19360            .await
19361            .unwrap();
19362        git::worktree_add_detached(&repo, &reviewer, "main")
19363            .await
19364            .unwrap();
19365        git::acquire_worktree_config(&repo).await.unwrap();
19366        state.enabled_worktree_config = true;
19367        state.status = RunStatus::Ready;
19368        state.candidates.push(Candidate {
19369            index: 0,
19370            label: 'A',
19371            agent: "implement".into(),
19372            branch: branch.clone(),
19373            worktree: candidate.clone(),
19374            summary: String::new(),
19375            stat: String::new(),
19376            files: 0,
19377            commits: 0,
19378            empty: false,
19379            failed: None,
19380            verified_noop: None,
19381            duration_ms: 0,
19382            folded: false,
19383        });
19384        state.save_under(&home).unwrap();
19385        let config_before = std::fs::read(common.join("config")).unwrap();
19386        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19387            .await
19388            .unwrap();
19389        // A separate OS handle bypasses the launcher's local gate, exactly as
19390        // an external updater does, and must still prevent all folding work.
19391        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19392        assert_eq!(
19393            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19394            0
19395        );
19396        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19397        assert!(
19398            tokio::time::timeout(Duration::from_millis(200), &mut fold)
19399                .await
19400                .is_err()
19401        );
19402        assert!(candidate.exists());
19403        assert!(reviewer.exists());
19404        assert!(git::branch_exists(&repo, &branch).await.unwrap());
19405        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19406        assert_eq!(
19407            git::git(&repo, &["worktree", "list", "--porcelain"])
19408                .await
19409                .unwrap(),
19410            worktrees_before
19411        );
19412        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19413            .await
19414            .unwrap()
19415            .unwrap_err();
19416        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19417        drop(fold);
19418        assert!(candidate.exists() && reviewer.exists());
19419        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19420        assert_eq!(
19421            state.resource_block.as_ref().unwrap().kind,
19422            "repository coordination"
19423        );
19424        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19425        drop(external);
19426        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19427        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19428            .await
19429            .unwrap()
19430            .unwrap();
19431        drop(fold);
19432        assert!(removed.contains(&branch));
19433        assert!(!root.exists());
19434        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19435        assert!(state.candidates[0].folded);
19436        assert!(!state.enabled_worktree_config);
19437        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19438        let enabled = git::git_raw(
19439            &repo,
19440            &["config", "--local", "--get", "extensions.worktreeConfig"],
19441        )
19442        .await
19443        .unwrap();
19444        assert!(!enabled.ok(), "the last config reference was released");
19445    }
19446
19447    /// A branch handed to a later run is that run's (and its pull request's):
19448    /// folding the run that released it must not delete it.
19449    #[tokio::test]
19450    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19451        let tmp = tempfile::tempdir().expect("tempdir");
19452        let repo = tmp.path().join("repo");
19453        std::fs::create_dir_all(&repo).unwrap();
19454        init_repo(&repo);
19455        let home = tmp.path().join("home");
19456
19457        let mut config = Config::default();
19458        config.graph.worktree_root = Some(tmp.path().join("wt"));
19459        let mut state = RunState::new(
19460            repo.clone(),
19461            "main".to_owned(),
19462            "deadbeef".to_owned(),
19463            "task".to_owned(),
19464            config,
19465        );
19466        // The worktree is already gone (released); the branch survives.
19467        git::git(&repo, &["branch", "magi/x/A", "main"])
19468            .await
19469            .expect("branch");
19470        state.candidates = vec![Candidate {
19471            index: 0,
19472            label: 'A',
19473            agent: "alpha".to_owned(),
19474            branch: "magi/x/A".to_owned(),
19475            worktree: state.worktree_root().join("cand-A"),
19476            summary: String::new(),
19477            stat: String::new(),
19478            files: 0,
19479            commits: 0,
19480            empty: false,
19481            failed: None,
19482            verified_noop: None,
19483            duration_ms: 0,
19484            folded: true,
19485        }];
19486        state.released_to = Some("20260901-000000-new1".to_owned());
19487        state.released_branches = vec!["magi/x/A".to_owned()];
19488
19489        fold_run(&mut state, true, &home).await.expect("fold_run");
19490        state.status = RunStatus::Merged;
19491        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19492        state.merge = Some(crate::run::MergeOutcome {
19493            mode: MergeMode::Pr,
19494            ok: true,
19495            detail: "confirmed merged".into(),
19496            empty: false,
19497        });
19498        fold_run(&mut state, true, &home)
19499            .await
19500            .expect("merged fold");
19501
19502        assert!(
19503            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19504            "the handed-over branch survives a fold"
19505        );
19506    }
19507
19508    /// A winner with nothing ahead of the base is caught before `gh` is ever
19509    /// asked for a pull request, and the message carries what the task's
19510    /// references resolved to.
19511    #[tokio::test]
19512    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19513        let tmp = tempfile::tempdir().expect("tempdir");
19514        let repo = tmp.path().join("repo");
19515        std::fs::create_dir_all(&repo).unwrap();
19516        init_repo(&repo);
19517        let run = |args: &[&str]| {
19518            let out = std::process::Command::new("git")
19519                .quiet()
19520                .args(args)
19521                .current_dir(&repo)
19522                .output()
19523                .expect("spawn git");
19524            assert!(out.status.success(), "git {args:?}");
19525        };
19526        run(&["branch", "magi/x/A"]);
19527        run(&["checkout", "-q", "-b", "magi/x/B"]);
19528        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19529        run(&["add", "-A"]);
19530        run(&["commit", "-q", "-m", "work"]);
19531        run(&["checkout", "-q", "main"]);
19532
19533        // A pull request is compared against the remote's base, so the
19534        // fixture needs one. Before it exists nothing can be read, and the
19535        // answer must be "not empty".
19536        let probe = RunState::new(
19537            repo.clone(),
19538            "main".to_owned(),
19539            "deadbeef".to_owned(),
19540            "task".to_owned(),
19541            Config::default(),
19542        );
19543        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19544        let bare = tmp.path().join("origin.git");
19545        let out = std::process::Command::new("git")
19546            .quiet()
19547            .args(["init", "-q", "--bare"])
19548            .arg(&bare)
19549            .output()
19550            .expect("spawn git");
19551        assert!(out.status.success(), "git init --bare");
19552        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19553        run(&["push", "-q", "origin", "main"]);
19554
19555        let mut state = RunState::new(
19556            repo.clone(),
19557            "main".to_owned(),
19558            "deadbeef".to_owned(),
19559            "task".to_owned(),
19560            Config::default(),
19561        );
19562        state.seeds = vec![refs::Seed {
19563            token: "magi/27b2/A".to_owned(),
19564            kind: refs::SeedKind::Unresolved,
19565            sha: String::new(),
19566            branch: true,
19567            detail: "no branch or commit named magi/27b2/A".to_owned(),
19568        }];
19569
19570        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19571        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19572        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19573        let detail = empty_candidate_detail(&state, "main");
19574        assert!(detail.starts_with("empty candidate"), "{detail}");
19575        assert!(!detail.contains("magi/27b2/A"), "{detail}");
19576        assert!(detail.contains("--start-from"), "{detail}");
19577    }
19578
19579    /// `status == Ready` used to be read as "this is the harmless
19580    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
19581    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
19582    /// run's PR was closed without merging — and reentering `merge` with
19583    /// `mode` still `Pr` does not know the difference, so it pushed and
19584    /// opened a second pull request. `mode == Local` reproduces the same
19585    /// blind spot without a network call: reentry must not attempt another
19586    /// git merge once this node has already recorded an outcome.
19587    #[tokio::test]
19588    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19589        let tmp = tempfile::tempdir().expect("tempdir");
19590        let repo = tmp.path().join("repo");
19591        std::fs::create_dir_all(&repo).unwrap();
19592        init_repo(&repo);
19593
19594        let mut config = Config::default();
19595        config.merge.mode = MergeMode::Local;
19596
19597        let mut state = RunState::new(
19598            repo.clone(),
19599            "main".to_owned(),
19600            "deadbeef".to_owned(),
19601            "task".to_owned(),
19602            config,
19603        );
19604        state.candidates = vec![Candidate {
19605            index: 0,
19606            label: 'A',
19607            agent: "alpha".to_owned(),
19608            branch: "does-not-exist".to_owned(),
19609            worktree: repo.clone(),
19610            summary: String::new(),
19611            stat: String::new(),
19612            files: 0,
19613            commits: 0,
19614            empty: false,
19615            failed: None,
19616            verified_noop: None,
19617            duration_ms: 0,
19618            folded: false,
19619        }];
19620        state.tally = Some(Tally {
19621            first_choice: BTreeMap::from([('A', 1)]),
19622            borda: BTreeMap::new(),
19623            winner: 'A',
19624            rankings: 1,
19625            unanimous_initial: true,
19626            deliberated: false,
19627            changed_votes: 0,
19628            unanimous_final: true,
19629            tie_break: None,
19630            judges: 0,
19631            present: 0,
19632            quorum: 0,
19633            met_quorum: true,
19634            uncontested: Some("only candidate A produced a change".to_owned()),
19635        });
19636        state.reviews = vec![ReviewRound {
19637            round: 1,
19638            head: "deadbeef".to_owned(),
19639            verified_head: None,
19640            verified_at: None,
19641            reviews: Vec::new(),
19642            e2e: Vec::new(),
19643            fix: None,
19644            blocking: 0,
19645            answered: 0,
19646            expected: 0,
19647            clean: true,
19648            verify_retried: false,
19649            e2e_deferred: false,
19650            e2e_defer_reason: None,
19651            progressed: false,
19652            vote_split: false,
19653            reconsideration: Vec::new(),
19654            verdict: None,
19655        }];
19656        state.gate = vec![CommandOutcome {
19657            command: "test".to_owned(),
19658            code: Some(0),
19659            output_tail: String::new(),
19660            duration_ms: 0,
19661            resource_blocked: false,
19662        }];
19663        state.gate_ran = true;
19664        // Reached its conclusion already — e.g. `land` closing the PR without
19665        // merging it, which (like the honest `MergeMode::None` path) leaves
19666        // `status` at `Ready`. The recorded outcome is what actually marks
19667        // this node done.
19668        state.status = RunStatus::Ready;
19669        state.merge = Some(MergeOutcome {
19670            mode: MergeMode::Local,
19671            ok: false,
19672            detail: "already concluded".to_owned(),
19673            empty: false,
19674        });
19675
19676        let mut runner = Runner {
19677            state,
19678            roles: ResolvedRoles {
19679                implementers: Vec::new(),
19680                judges: Vec::new(),
19681                reviewers: Vec::new(),
19682                fixer: None,
19683                conductor: conductor(),
19684                implementer_roster: Vec::new(),
19685                judge_roster: Vec::new(),
19686                reviewer_roster: Vec::new(),
19687            },
19688            sem: Arc::new(Semaphore::new(1)),
19689            pause: Pause::new(),
19690            interrupt: Pause::new(),
19691        };
19692
19693        runner.merge().await.expect("merge");
19694
19695        assert_eq!(
19696            runner.state.status,
19697            RunStatus::Ready,
19698            "a concluded run's status must not change on reentry"
19699        );
19700        assert_eq!(
19701            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19702            Some("already concluded"),
19703            "merge must not run again once the node already recorded an outcome"
19704        );
19705    }
19706
19707    /// `gate` leaves `state.gate_ran` false both before it has ever run and
19708    /// when its last attempt was resource-blocked (the shared build cache
19709    /// could not be acquired or confirmed fresh in time - see
19710    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
19711    /// `Vec` this also leaves behind used to read as "nothing failed" and let
19712    /// a run merge a tree the gate never actually checked - exactly the case
19713    /// a contended cache produces on every retry until it clears. `merge`
19714    /// must refuse until `gate` has actually recorded an attempt.
19715    #[tokio::test]
19716    async fn merge_refuses_a_gate_that_has_not_actually_run() {
19717        let tmp = tempfile::tempdir().expect("tempdir");
19718        let repo = tmp.path().join("repo");
19719        std::fs::create_dir_all(&repo).unwrap();
19720        init_repo(&repo);
19721
19722        let mut config = Config::default();
19723        config.merge.mode = MergeMode::Local;
19724
19725        let mut state = RunState::new(
19726            repo.clone(),
19727            "main".to_owned(),
19728            "deadbeef".to_owned(),
19729            "task".to_owned(),
19730            config,
19731        );
19732        state.candidates = vec![Candidate {
19733            index: 0,
19734            label: 'A',
19735            agent: "alpha".to_owned(),
19736            branch: "does-not-exist".to_owned(),
19737            worktree: repo.clone(),
19738            summary: String::new(),
19739            stat: String::new(),
19740            files: 0,
19741            commits: 0,
19742            empty: false,
19743            failed: None,
19744            verified_noop: None,
19745            duration_ms: 0,
19746            folded: false,
19747        }];
19748        state.tally = Some(Tally {
19749            first_choice: BTreeMap::from([('A', 1)]),
19750            borda: BTreeMap::new(),
19751            winner: 'A',
19752            rankings: 1,
19753            unanimous_initial: true,
19754            deliberated: false,
19755            changed_votes: 0,
19756            unanimous_final: true,
19757            tie_break: None,
19758            judges: 0,
19759            present: 0,
19760            quorum: 0,
19761            met_quorum: true,
19762            uncontested: Some("only candidate A produced a change".to_owned()),
19763        });
19764        state.reviews = vec![ReviewRound {
19765            round: 1,
19766            head: "deadbeef".to_owned(),
19767            verified_head: None,
19768            verified_at: None,
19769            reviews: Vec::new(),
19770            e2e: Vec::new(),
19771            fix: None,
19772            blocking: 0,
19773            answered: 0,
19774            expected: 0,
19775            clean: true,
19776            verify_retried: false,
19777            e2e_deferred: false,
19778            e2e_defer_reason: None,
19779            progressed: false,
19780            vote_split: false,
19781            reconsideration: Vec::new(),
19782            verdict: None,
19783        }];
19784        // The point: `gate` has not recorded anything yet.
19785        state.gate = Vec::new();
19786        state.gate_ran = false;
19787        state.status = RunStatus::Gating;
19788
19789        let mut runner = Runner {
19790            state,
19791            roles: ResolvedRoles {
19792                implementers: Vec::new(),
19793                judges: Vec::new(),
19794                reviewers: Vec::new(),
19795                fixer: None,
19796                conductor: conductor(),
19797                implementer_roster: Vec::new(),
19798                judge_roster: Vec::new(),
19799                reviewer_roster: Vec::new(),
19800            },
19801            sem: Arc::new(Semaphore::new(1)),
19802            pause: Pause::new(),
19803            interrupt: Pause::new(),
19804        };
19805
19806        runner.merge().await.expect("merge");
19807
19808        assert!(
19809            runner.state.merge.is_none(),
19810            "an empty gate must never be read as a passing one: {:?}",
19811            runner.state.merge
19812        );
19813    }
19814
19815    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
19816    /// commands configured and `merge.mode` is `none` (a review-only run).
19817    /// `gate` must still record a real attempt — zero commands, vacuously
19818    /// passed — rather than leaving `state.gate` empty in a way `merge`
19819    /// cannot tell apart from "never ran"; otherwise the run reaches
19820    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
19821    #[tokio::test]
19822    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19823        let tmp = tempfile::tempdir().expect("tempdir");
19824        let repo = tmp.path().join("repo");
19825        std::fs::create_dir_all(&repo).unwrap();
19826        init_repo(&repo);
19827
19828        // Default config: `verify.gate` empty, `merge.mode` is `none`.
19829        let config = Config::default();
19830
19831        let mut state = RunState::new(
19832            repo.clone(),
19833            "main".to_owned(),
19834            "deadbeef".to_owned(),
19835            "task".to_owned(),
19836            config,
19837        );
19838        state.candidates = vec![Candidate {
19839            index: 0,
19840            label: 'A',
19841            agent: "alpha".to_owned(),
19842            branch: "does-not-exist".to_owned(),
19843            worktree: repo.clone(),
19844            summary: String::new(),
19845            stat: String::new(),
19846            files: 0,
19847            commits: 0,
19848            empty: false,
19849            failed: None,
19850            verified_noop: None,
19851            duration_ms: 0,
19852            folded: false,
19853        }];
19854        state.tally = Some(Tally {
19855            first_choice: BTreeMap::from([('A', 1)]),
19856            borda: BTreeMap::new(),
19857            winner: 'A',
19858            rankings: 1,
19859            unanimous_initial: true,
19860            deliberated: false,
19861            changed_votes: 0,
19862            unanimous_final: true,
19863            tie_break: None,
19864            judges: 0,
19865            present: 0,
19866            quorum: 0,
19867            met_quorum: true,
19868            uncontested: Some("only candidate A produced a change".to_owned()),
19869        });
19870        state.reviews = vec![ReviewRound {
19871            round: 1,
19872            head: "deadbeef".to_owned(),
19873            verified_head: None,
19874            verified_at: None,
19875            reviews: Vec::new(),
19876            e2e: Vec::new(),
19877            fix: None,
19878            blocking: 0,
19879            answered: 0,
19880            expected: 0,
19881            clean: true,
19882            verify_retried: false,
19883            e2e_deferred: false,
19884            e2e_defer_reason: None,
19885            progressed: false,
19886            vote_split: false,
19887            reconsideration: Vec::new(),
19888            verdict: None,
19889        }];
19890
19891        let mut runner = Runner {
19892            state,
19893            roles: ResolvedRoles {
19894                implementers: Vec::new(),
19895                judges: Vec::new(),
19896                reviewers: Vec::new(),
19897                fixer: None,
19898                conductor: conductor(),
19899                implementer_roster: Vec::new(),
19900                judge_roster: Vec::new(),
19901                reviewer_roster: Vec::new(),
19902            },
19903            sem: Arc::new(Semaphore::new(1)),
19904            pause: Pause::new(),
19905            interrupt: Pause::new(),
19906        };
19907
19908        runner.gate().await.expect("gate");
19909        assert!(
19910            runner.state.gate_ran,
19911            "zero configured commands is still a real attempt, not an unrun gate"
19912        );
19913        assert!(runner.state.gate.is_empty());
19914        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19915        assert_ne!(
19916            runner.state.status,
19917            RunStatus::Blocked,
19918            "a gate with nothing to check must not read as failed"
19919        );
19920
19921        runner.merge().await.expect("merge");
19922        assert_eq!(
19923            runner.state.status,
19924            RunStatus::Ready,
19925            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19926        );
19927    }
19928
19929    /// `Config::cache_dir` is derived from `verify.e2e` as well as
19930    /// `verify.gate` (so the e2e leg and the final gate never build against
19931    /// different directories). With zero `verify.gate` commands but a
19932    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
19933    /// that lease before discovering it had nothing to run - so a repo with
19934    /// no gate commands could come back `resource_blocked` (and therefore
19935    /// still `gate_ran == false`) on nothing but an unrelated run holding the
19936    /// cache, exactly the contention this run's own zero commands could
19937    /// never have touched. `gate` must recognise there is nothing to check
19938    /// before it ever asks for the lease.
19939    #[tokio::test]
19940    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19941        crate::run::pin_test_home();
19942        let home = crate::run::home();
19943
19944        let tmp = tempfile::tempdir().expect("tempdir");
19945        let repo = tmp.path().join("repo");
19946        std::fs::create_dir_all(&repo).unwrap();
19947        init_repo(&repo);
19948        // Unique to this test, so holding its lease cannot collide with
19949        // another test sharing the same process-wide `home`.
19950        let cache_dir = tmp.path().join("target");
19951
19952        let mut config = Config::default();
19953        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19954        // `verify.gate` stays empty (the default). Any accidental cache
19955        // acquisition must report contention immediately, regardless of how
19956        // long unrelated Git subprocesses take under parallel test load.
19957        config.graph.timeout_verify = Some(2);
19958        config.graph.timeout_resource_wait = 0;
19959
19960        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19961        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19962            .expect("no io error acquiring directly")
19963        {
19964            crate::cache::AcquireOutcome::Acquired(g) => g,
19965            crate::cache::AcquireOutcome::Busy(b) => {
19966                panic!("expected the direct acquire to win the lease first: {b:?}")
19967            }
19968        };
19969
19970        let mut state = RunState::new(
19971            repo.clone(),
19972            "main".to_owned(),
19973            "deadbeef".to_owned(),
19974            "task".to_owned(),
19975            config,
19976        );
19977        state.candidates = vec![Candidate {
19978            index: 0,
19979            label: 'A',
19980            agent: "alpha".to_owned(),
19981            branch: "does-not-exist".to_owned(),
19982            worktree: repo.clone(),
19983            summary: String::new(),
19984            stat: String::new(),
19985            files: 0,
19986            commits: 0,
19987            empty: false,
19988            failed: None,
19989            verified_noop: None,
19990            duration_ms: 0,
19991            folded: false,
19992        }];
19993        state.tally = Some(Tally {
19994            first_choice: BTreeMap::from([('A', 1)]),
19995            borda: BTreeMap::new(),
19996            winner: 'A',
19997            rankings: 1,
19998            unanimous_initial: true,
19999            deliberated: false,
20000            changed_votes: 0,
20001            unanimous_final: true,
20002            tie_break: None,
20003            judges: 0,
20004            present: 0,
20005            quorum: 0,
20006            met_quorum: true,
20007            uncontested: Some("only candidate A produced a change".to_owned()),
20008        });
20009        state.reviews = vec![ReviewRound {
20010            round: 1,
20011            head: "deadbeef".to_owned(),
20012            verified_head: None,
20013            verified_at: None,
20014            reviews: Vec::new(),
20015            e2e: Vec::new(),
20016            fix: None,
20017            blocking: 0,
20018            answered: 0,
20019            expected: 0,
20020            clean: true,
20021            verify_retried: false,
20022            e2e_deferred: false,
20023            e2e_defer_reason: None,
20024            progressed: false,
20025            vote_split: false,
20026            reconsideration: Vec::new(),
20027            verdict: None,
20028        }];
20029
20030        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20031        state.reviews[0].e2e = vec![CommandOutcome {
20032            command: state.config.verify.e2e[0].clone(),
20033            code: Some(0),
20034            output_tail: String::new(),
20035            duration_ms: 0,
20036            resource_blocked: false,
20037        }];
20038        let mut runner = Runner {
20039            state,
20040            roles: ResolvedRoles {
20041                implementers: Vec::new(),
20042                judges: Vec::new(),
20043                reviewers: Vec::new(),
20044                fixer: None,
20045                conductor: conductor(),
20046                implementer_roster: Vec::new(),
20047                judge_roster: Vec::new(),
20048                reviewer_roster: Vec::new(),
20049            },
20050            sem: Arc::new(Semaphore::new(1)),
20051            pause: Pause::new(),
20052            interrupt: Pause::new(),
20053        };
20054
20055        runner.gate().await.expect("gate");
20056        assert!(runner.state.resource_block.is_none());
20057        assert!(
20058            !runner
20059                .state
20060                .events
20061                .iter()
20062                .any(|event| { event.message.contains("waiting for the shared build cache") }),
20063            "a gate with nothing to run must never request the held cache lease"
20064        );
20065        assert!(
20066            runner.state.gate_ran,
20067            "zero commands is still a real, immediate attempt"
20068        );
20069        assert!(runner.state.gate.is_empty());
20070        assert_ne!(
20071            runner.state.status,
20072            RunStatus::Blocked,
20073            "must not read as resource-blocked on a lease it never asked for"
20074        );
20075    }
20076
20077    /// A separate process carries a live worker's environment throughout all
20078    /// verification entry points; the parallel harness never changes its env.
20079    #[cfg(unix)]
20080    #[tokio::test]
20081    async fn verification_is_standalone_inside_a_live_worker() {
20082        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20083        if std::env::var_os(CHILD).is_none() {
20084            let home = tempfile::tempdir().unwrap();
20085            let generation = crate::worker::pin_current(home.path()).unwrap();
20086            let output = std::process::Command::new(std::env::current_exe().unwrap())
20087                .args([
20088                    "--exact",
20089                    "graph::tests::verification_is_standalone_inside_a_live_worker",
20090                    "--nocapture",
20091                ])
20092                .env(CHILD, "1")
20093                .env("MAGI_HOME", home.path())
20094                .env("MAGI_WORKER_HOME", home.path())
20095                .env("MAGI_WORKER_ID", "abcdef")
20096                .env("MAGI_WORKER_GENERATION", &generation.id)
20097                .env("MAGI_WORKER_EXE", &generation.executable)
20098                .env("MAGI_RUN", "verification-parent")
20099                .env("MAGI_NODE", "gate")
20100                .env("MAGI_SEAT", "worker-fixture")
20101                .env("MAGI_PROCESS_OWNER", "fixture-owner")
20102                .env("MAGI_NO_AUTOUPDATE", "0")
20103                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
20104                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
20105                .env(
20106                    "VERIFY_EXPECT_TOOLCHAIN",
20107                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
20108                )
20109                .env(
20110                    "VERIFY_EXPECT_CACHE",
20111                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
20112                )
20113                .output()
20114                .unwrap();
20115            assert!(
20116                output.status.success()
20117                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
20118                "{}\n{}",
20119                String::from_utf8_lossy(&output.stdout),
20120                String::from_utf8_lossy(&output.stderr)
20121            );
20122            return;
20123        }
20124        let home = crate::run::home();
20125        let directory = home.join("workers/jobs/abcdef");
20126        std::fs::create_dir_all(&directory).unwrap();
20127        let job = crate::worker::Job {
20128            id: "abcdef".into(),
20129            sequence: 1,
20130            kind: "task".into(),
20131            key: "task:verification".into(),
20132            payload: serde_json::json!({}),
20133            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
20134            version: Some(env!("CARGO_PKG_VERSION").into()),
20135            status: crate::worker::Status::Running,
20136            result: None,
20137            pid: Some(std::process::id()),
20138            created_at: 0,
20139            finished_at: None,
20140        };
20141        std::fs::write(
20142            directory.join("job.json"),
20143            serde_json::to_vec(&job).unwrap(),
20144        )
20145        .unwrap();
20146        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
20147        let assert_worker = || {
20148            assert_eq!(
20149                crate::worker::current_job(&home).unwrap().unwrap().id,
20150                job.id
20151            );
20152            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
20153            assert_eq!(
20154                crate::worker::helper_executable(&home).unwrap(),
20155                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20156            );
20157            assert!(
20158                crate::updater::Checker::new(&crate::config::Update {
20159                    mode: crate::config::UpdateMode::Notify,
20160                    interval: None,
20161                    auto_restart: false,
20162                })
20163                .is_none(),
20164                "the driver remains pinned, even while tests are standalone"
20165            );
20166        };
20167        assert_worker();
20168        let repo = home.join("repo");
20169        std::fs::create_dir_all(&repo).unwrap();
20170        init_repo(&repo);
20171        // Keep storage, seat/control context, containment and tool lookup.
20172        // No inherited values (or credentials) are printed on success.
20173        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20174            test -z "${MAGI_WORKER_GENERATION+x}" &&
20175            test -z "${MAGI_WORKER_HOME+x}" &&
20176            test -z "${MAGI_WORKER_EXE+x}" &&
20177            test "$MAGI_HOME/repo" -ef . &&
20178            test "$MAGI_RUN" = verification-parent &&
20179            test "$MAGI_NODE" = gate &&
20180            test "$MAGI_SEAT" = worker-fixture &&
20181            test -n "$MAGI_PROCESS_OWNER" &&
20182            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20183            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20184            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20185            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20186        // Negative control: an ordinary child inherits the contamination.
20187        assert!(
20188            !std::process::Command::new("sh")
20189                .args(["-c", probe])
20190                .current_dir(&repo)
20191                .status()
20192                .unwrap()
20193                .success()
20194        );
20195        let unrelated_home = tempfile::tempdir().unwrap();
20196        assert!(
20197            crate::worker::current_job(unrelated_home.path()).is_err(),
20198            "an inherited worker id cannot be resolved in a unit fixture's home"
20199        );
20200        let mut runner = runner_at(RunStatus::Gating);
20201        runner.state.repo = repo.clone();
20202        runner.state.config.disk.min_free_bytes = 0;
20203        runner.state.config.coordination = crate::config::test_coordination();
20204        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20205        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20206        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20207        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20208        let winner = Candidate {
20209            index: 0,
20210            label: 'A',
20211            agent: "fixture".into(),
20212            branch: "main".into(),
20213            worktree: repo.clone(),
20214            summary: String::new(),
20215            stat: String::new(),
20216            files: 0,
20217            commits: 0,
20218            empty: false,
20219            failed: None,
20220            verified_noop: None,
20221            duration_ms: 0,
20222            folded: false,
20223        };
20224        let (e2e, retried, pids) = run_e2e_with_retry(
20225            &mut runner.state,
20226            &["sh".into(), "-c".into()],
20227            &[format!("{probe} && printf initial")],
20228            &repo,
20229            Duration::from_secs(10),
20230            "initial",
20231        )
20232        .await;
20233        assert!(!retried && pids.is_empty());
20234        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20235        assert_eq!(e2e[0].output_tail, "initial");
20236        let gate = runner.run_gate(&winner).await.unwrap();
20237        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20238        // An interrupted repair must recheck e2e from saved state. Landing
20239        // reuses that state and must run both pre-gate and gate again.
20240        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20241            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20242            passed: false,
20243            outcomes: Vec::new(),
20244        });
20245        runner.state.save().unwrap();
20246        let mut resumed = RunState::load(&runner.state.id).unwrap();
20247        let wait = crate::repo_lock::WaitPolicy::default();
20248        assert!(matches!(
20249            check_gate_repair_e2e(&mut resumed, &winner, &wait)
20250                .await
20251                .unwrap(),
20252            GateFix::Retry
20253        ));
20254        let saved = RunState::load(&resumed.id).unwrap();
20255        let repair = saved.gate_repair_e2e.unwrap();
20256        assert!(repair.passed);
20257        assert_eq!(repair.outcomes.len(), 1);
20258        assert!(repair.outcomes[0].ok());
20259        let landing = run_landing_gate(&mut resumed, &winner, &wait)
20260            .await
20261            .unwrap();
20262        assert_eq!(landing.len(), 1);
20263        assert!(landing[0].ok(), "{landing:?}");
20264        assert_eq!(
20265            std::fs::read_to_string(repo.join("phases")).unwrap(),
20266            "pre\ngate\ne2e\npre\ngate\n"
20267        );
20268        assert_worker();
20269    }
20270
20271    #[cfg(unix)]
20272    #[tokio::test]
20273    async fn verification_keeps_the_panic_and_the_full_output() {
20274        crate::run::pin_test_home();
20275        let tmp = tempfile::tempdir().unwrap();
20276        let mut config = Config::default();
20277        config.disk.min_free_bytes = 0;
20278        let mut state = RunState::new(
20279            tmp.path().to_owned(),
20280            "main".into(),
20281            "base".into(),
20282            "verification evidence".into(),
20283            config,
20284        );
20285        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20286                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
20287                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20288        let mut names = Vec::new();
20289        for _ in 0..2 {
20290            let (outcomes, _) = run_commands(
20291                &mut state,
20292                "gate",
20293                "gate",
20294                0,
20295                &["sh".into(), "-c".into()],
20296                &[script.into()],
20297                tmp.path(),
20298                Duration::from_secs(60),
20299            )
20300            .await;
20301            let tail = &outcomes[0].output_tail;
20302            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20303            assert!(tail.contains("boom assertion"));
20304            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20305            let line = tail.lines().next().unwrap();
20306            names.push(line.to_owned());
20307        }
20308        assert_ne!(names[0], names[1], "the second run must not overwrite");
20309        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20310        assert!(full.len() > 500_000 && full.contains("boom assertion"));
20311        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20312
20313        // Storage failures are evidence too; preserve the command's verdict.
20314        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20315        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20316        let (outcomes, _) = run_commands(
20317            &mut state,
20318            "gate",
20319            "gate",
20320            0,
20321            &["sh".into(), "-c".into()],
20322            &["echo failure-output; exit 7".into()],
20323            tmp.path(),
20324            Duration::from_secs(10),
20325        )
20326        .await;
20327        assert_eq!(outcomes[0].code, Some(7));
20328        assert!(state.events.iter().any(|e| {
20329            e.node == "gate" && e.message.contains("could not save the full gate output:")
20330        }));
20331    }
20332
20333    #[cfg(unix)]
20334    #[tokio::test]
20335    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20336        crate::run::pin_test_home();
20337        let tmp = tempfile::tempdir().unwrap();
20338        let mut config = Config::default();
20339        config.disk.min_free_bytes = 0;
20340        let mut state = RunState::new(
20341            tmp.path().to_owned(),
20342            "main".into(),
20343            "base".into(),
20344            "verification evidence".into(),
20345            config,
20346        );
20347        let (outcomes, pids) = run_commands(
20348            &mut state,
20349            "gate",
20350            "gate",
20351            1,
20352            &["sh".into(), "-c".into()],
20353            &[
20354                "printf before-timeout; exec sleep 30".into(),
20355                "kill -TERM $$".into(),
20356                "exit 7".into(),
20357                "printf success".into(),
20358            ],
20359            tmp.path(),
20360            Duration::from_secs(1),
20361        )
20362        .await;
20363        wait_for_timed_out_children_to_die(&pids).await;
20364        assert_eq!(
20365            pids.len(),
20366            1,
20367            "unleased timeout retains its death-check PID"
20368        );
20369        assert_eq!(outcomes.len(), 4);
20370        assert_eq!(outcomes[0].code, None);
20371        assert!(outcomes[0].output_tail.contains("before-timeout"));
20372        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20373        assert_eq!(outcomes[1].code, None);
20374        assert!(outcomes[1].output_tail.contains("command terminated:"));
20375        assert!(!outcomes[1].output_tail.contains("timed out"));
20376        assert_eq!(outcomes[2].code, Some(7));
20377        assert!(!outcomes[2].output_tail.contains("timed out"));
20378        assert_eq!(outcomes[3].code, Some(0));
20379        assert_eq!(outcomes[3].output_tail, "success");
20380        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20381    }
20382
20383    /// The addendum's second gap: a `verify.gate` command running for real
20384    /// wall-clock time had nothing at all to show for it in `active` before
20385    /// `run_commands` learned to record it — a run could sit in `Gating` for
20386    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
20387    /// what was actually happening. Proven with a genuinely still-running
20388    /// command, not just a before/after check on the final state: a poller
20389    /// task reads the same `run.json` `gate()` is writing, the same way the
20390    /// phone or `magi show` would, while the shell command is still blocked
20391    /// on its own release marker.
20392    #[tokio::test]
20393    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20394        crate::run::pin_test_home();
20395
20396        let tmp = tempfile::tempdir().expect("tempdir");
20397        let repo = tmp.path().join("repo");
20398        std::fs::create_dir_all(&repo).unwrap();
20399        init_repo(&repo);
20400
20401        let mut config = Config::default();
20402        config.disk.min_free_bytes = 0;
20403        config.verify.gate = vec![
20404            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20405             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20406                .to_owned(),
20407        ];
20408
20409        let mut state = RunState::new(
20410            repo.clone(),
20411            "main".to_owned(),
20412            "deadbeef".to_owned(),
20413            "task".to_owned(),
20414            config,
20415        );
20416        let run_id = state.id.clone();
20417        state.candidates = vec![Candidate {
20418            index: 0,
20419            label: 'A',
20420            agent: "alpha".to_owned(),
20421            branch: "does-not-exist".to_owned(),
20422            worktree: repo.clone(),
20423            summary: String::new(),
20424            stat: String::new(),
20425            files: 0,
20426            commits: 0,
20427            empty: false,
20428            failed: None,
20429            verified_noop: None,
20430            duration_ms: 0,
20431            folded: false,
20432        }];
20433        state.tally = Some(Tally {
20434            first_choice: BTreeMap::from([('A', 1)]),
20435            borda: BTreeMap::new(),
20436            winner: 'A',
20437            rankings: 1,
20438            unanimous_initial: true,
20439            deliberated: false,
20440            changed_votes: 0,
20441            unanimous_final: true,
20442            tie_break: None,
20443            judges: 0,
20444            present: 0,
20445            quorum: 0,
20446            met_quorum: true,
20447            uncontested: Some("only candidate A produced a change".to_owned()),
20448        });
20449        state.reviews = vec![ReviewRound {
20450            round: 1,
20451            head: "deadbeef".to_owned(),
20452            verified_head: None,
20453            verified_at: None,
20454            reviews: Vec::new(),
20455            e2e: Vec::new(),
20456            fix: None,
20457            blocking: 0,
20458            answered: 0,
20459            expected: 0,
20460            clean: true,
20461            verify_retried: false,
20462            e2e_deferred: false,
20463            e2e_defer_reason: None,
20464            progressed: false,
20465            vote_split: false,
20466            reconsideration: Vec::new(),
20467            verdict: None,
20468        }];
20469
20470        let mut runner = Runner {
20471            state,
20472            roles: ResolvedRoles {
20473                implementers: Vec::new(),
20474                judges: Vec::new(),
20475                reviewers: Vec::new(),
20476                fixer: None,
20477                conductor: conductor(),
20478                implementer_roster: Vec::new(),
20479                judge_roster: Vec::new(),
20480                reviewer_roster: Vec::new(),
20481            },
20482            sem: Arc::new(Semaphore::new(1)),
20483            pause: Pause::new(),
20484            interrupt: Pause::new(),
20485        };
20486
20487        let started_marker = repo.join("started.marker");
20488        let release_marker = repo.join("release.marker");
20489        let poller = tokio::spawn(async move {
20490            // Bounded so a regression that never records the task entry
20491            // fails this test in seconds instead of hanging the suite —
20492            // the same shape `a_park_requested_while_a_seat_is_mid_call_
20493            // does_not_cut_it_short` uses for the same reason.
20494            for _ in 0..100 {
20495                if started_marker.exists()
20496                    && let Ok(s) = crate::run::RunState::load(&run_id)
20497                    && let Some(a) = s.active.get("gate")
20498                {
20499                    std::fs::write(&release_marker, b"go").expect("release marker");
20500                    return Some(a.clone());
20501                }
20502                tokio::time::sleep(Duration::from_millis(50)).await;
20503            }
20504            None
20505        });
20506
20507        runner.gate().await.expect("gate");
20508        let captured = poller.await.expect("poller task");
20509        let captured = captured.expect(
20510            "the poller never saw a `gate` task entry in run.json while the command was \
20511             still blocked on its own release marker",
20512        );
20513
20514        assert_eq!(captured.task.as_deref(), Some("gate"));
20515        assert_eq!(captured.node, "gate");
20516        assert_eq!(captured.index, Some(1));
20517        assert_eq!(captured.total, Some(1));
20518        assert!(
20519            captured
20520                .command
20521                .as_deref()
20522                .is_some_and(|c| c.contains("started.marker")),
20523            "{captured:?}"
20524        );
20525
20526        assert!(
20527            runner.state.active.is_empty(),
20528            "the entry must be cleared once the command actually finished: {:?}",
20529            runner.state.active
20530        );
20531        assert!(runner.state.gate_ran);
20532        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20533    }
20534
20535    /// The hand-off over a blocking finding a reviewer rejected on leaves a
20536    /// record for `land`; one with only a Minor, or no reject, leaves none.
20537    #[tokio::test]
20538    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20539        use crate::verdict::{Finding, ReviewVote, Severity};
20540        crate::run::pin_test_home();
20541        let tmp = tempfile::tempdir().expect("tempdir");
20542        let repo = tmp.path().join("repo");
20543        std::fs::create_dir_all(&repo).unwrap();
20544        init_repo(&repo);
20545
20546        for (severity, vote, expect) in [
20547            (Severity::Major, ReviewVote::Reject, true),
20548            (Severity::Minor, ReviewVote::Reject, false),
20549            (Severity::Major, ReviewVote::Approve, false),
20550        ] {
20551            let mut round = review_round(false, 1, 1, 1, false, true);
20552            round.reviews = vec![ReviewRecord {
20553                reviewer: 1,
20554                agent: "alpha".to_owned(),
20555                summary: String::new(),
20556                findings: vec![Finding {
20557                    id: "R1-1-1".to_owned(),
20558                    severity,
20559                    file: None,
20560                    line: None,
20561                    title: "t".to_owned(),
20562                    detail: String::new(),
20563                }],
20564                vote: Some(vote),
20565                failed: None,
20566                duration_ms: 0,
20567                attempts: 0,
20568            }];
20569            let mut state = RunState::new(
20570                repo.clone(),
20571                "main".to_owned(),
20572                "deadbeef".to_owned(),
20573                "task".to_owned(),
20574                Config::default(),
20575            );
20576            state.reviews = vec![round];
20577            let mut runner = Runner {
20578                state,
20579                roles: ResolvedRoles {
20580                    implementers: Vec::new(),
20581                    judges: Vec::new(),
20582                    reviewers: Vec::new(),
20583                    fixer: None,
20584                    conductor: conductor(),
20585                    implementer_roster: Vec::new(),
20586                    judge_roster: Vec::new(),
20587                    reviewer_roster: Vec::new(),
20588                },
20589                sem: Arc::new(Semaphore::new(1)),
20590                pause: Pause::new(),
20591                interrupt: Pause::new(),
20592            };
20593            let shell = runner.state.config.shell();
20594            runner
20595                .stop_reviewing("round budget spent", &shell, &repo)
20596                .await
20597                .expect("stop_reviewing");
20598            assert_eq!(runner.state.status, RunStatus::Gating);
20599            assert_eq!(
20600                runner.state.contested_handoff.is_some(),
20601                expect,
20602                "{severity:?} + {vote:?}"
20603            );
20604        }
20605    }
20606
20607    /// The shape the incident this whole fix responds to actually had: the
20608    /// round budget spent, the last round's own e2e blocked on the shared
20609    /// build cache (held here by a live pid — this test process — exactly
20610    /// `cache`'s own unit tests' pattern for "another owner, still alive"
20611    /// without forking a process). `stop_reviewing` must retry it — not
20612    /// silently leave the round looking untouched (the catch-up-only half of
20613    /// the bug), and not read the contention as a red `e2e` and block the
20614    /// run on it (the other half). Called directly, the same way
20615    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
20616    /// above exercises `gate`, so this never needs a real cargo build to
20617    /// reach: the lease is never released, so `with_cache_lease` never gets
20618    /// past acquiring it into anything that would need a real workspace.
20619    #[tokio::test]
20620    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20621        crate::run::pin_test_home();
20622        let home = crate::run::home();
20623
20624        let tmp = tempfile::tempdir().expect("tempdir");
20625        let repo = tmp.path().join("repo");
20626        std::fs::create_dir_all(&repo).unwrap();
20627        init_repo(&repo);
20628        let head = crate::git::rev_parse(&repo, "HEAD")
20629            .await
20630            .expect("rev-parse");
20631        // Unique to this test, so holding its lease cannot collide with
20632        // another test sharing the same process-wide `home`.
20633        let cache_dir = tmp.path().join("target");
20634
20635        let mut config = Config::default();
20636        config.verify.e2e = vec![format!(
20637            "CARGO_TARGET_DIR='{}' test -f README.md",
20638            cache_dir.display()
20639        )];
20640        config.graph.review_rounds = 1;
20641        // Bounded so a regression that does start waiting fails the test in
20642        // seconds, not hangs it.
20643        config.graph.timeout_verify = Some(2);
20644        config.graph.timeout_resource_wait = 2;
20645
20646        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20647        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20648            .expect("no io error acquiring directly")
20649        {
20650            crate::cache::AcquireOutcome::Acquired(g) => g,
20651            crate::cache::AcquireOutcome::Busy(b) => {
20652                panic!("expected the direct acquire to win the lease first: {b:?}")
20653            }
20654        };
20655
20656        let mut state = RunState::new(
20657            repo.clone(),
20658            "main".to_owned(),
20659            head.clone(),
20660            "task".to_owned(),
20661            config,
20662        );
20663        state.candidates = vec![Candidate {
20664            index: 0,
20665            label: 'A',
20666            agent: "alpha".to_owned(),
20667            branch: "does-not-exist".to_owned(),
20668            worktree: repo.clone(),
20669            summary: String::new(),
20670            stat: String::new(),
20671            files: 0,
20672            commits: 0,
20673            empty: false,
20674            failed: None,
20675            verified_noop: None,
20676            duration_ms: 0,
20677            folded: false,
20678        }];
20679        state.tally = Some(Tally {
20680            first_choice: BTreeMap::from([('A', 1)]),
20681            borda: BTreeMap::new(),
20682            winner: 'A',
20683            rankings: 1,
20684            unanimous_initial: true,
20685            deliberated: false,
20686            changed_votes: 0,
20687            unanimous_final: true,
20688            tie_break: None,
20689            judges: 0,
20690            present: 0,
20691            quorum: 0,
20692            met_quorum: true,
20693            uncontested: Some("only candidate A produced a change".to_owned()),
20694        });
20695        // The round budget's last round, deferred: `needs_catchup_run`'s
20696        // other trigger. `stop_reviewing`'s retry machinery must treat this
20697        // exactly like a resource-blocked attempt once it actually runs.
20698        state.reviews = vec![ReviewRound {
20699            round: 1,
20700            head: head.clone(),
20701            verified_head: None,
20702            verified_at: None,
20703            reviews: Vec::new(),
20704            e2e: Vec::new(),
20705            fix: None,
20706            blocking: 1,
20707            answered: 1,
20708            expected: 1,
20709            clean: false,
20710            verify_retried: false,
20711            e2e_deferred: true,
20712            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20713            progressed: false,
20714            vote_split: false,
20715            reconsideration: Vec::new(),
20716            verdict: None,
20717        }];
20718
20719        let mut runner = Runner {
20720            state,
20721            roles: ResolvedRoles {
20722                implementers: Vec::new(),
20723                judges: Vec::new(),
20724                reviewers: Vec::new(),
20725                fixer: None,
20726                conductor: conductor(),
20727                implementer_roster: Vec::new(),
20728                judge_roster: Vec::new(),
20729                reviewer_roster: Vec::new(),
20730            },
20731            sem: Arc::new(Semaphore::new(1)),
20732            pause: Pause::new(),
20733            interrupt: Pause::new(),
20734        };
20735
20736        let shell = runner.state.config.shell();
20737        runner
20738            .stop_reviewing("round budget spent", &shell, &repo)
20739            .await
20740            .expect("stop_reviewing");
20741
20742        let last = runner.state.reviews.last().expect("round record");
20743        assert_eq!(
20744            last.e2e_status(),
20745            E2eStatus::ResourceBlocked,
20746            "the shared cache is still held; the attempt must read as blocked, not deferred or \
20747             failed: {last:?}"
20748        );
20749        assert_eq!(
20750            last.verified_head.as_deref(),
20751            Some(head.as_str()),
20752            "which commit this attempt targeted is known even though nothing finished checking \
20753             it"
20754        );
20755        let first_attempt_at = last
20756            .verified_at
20757            .expect("when this attempt ran is known too");
20758        assert_ne!(
20759            runner.state.status,
20760            RunStatus::Blocked,
20761            "contention is evidence about the machine, not the patch — it must not settle the \
20762             run as blocked: {:?}",
20763            runner.state.status
20764        );
20765        assert!(
20766            !runner
20767                .state
20768                .events
20769                .iter()
20770                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20771            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20772            runner.state.events
20773        );
20774
20775        // The cache is still held: a later reentry must retry the same
20776        // round's verification again — not leave it looking exactly as
20777        // untouched as the first blocked attempt, which is indistinguishable
20778        // from never having tried again at all.
20779        runner
20780            .stop_reviewing("round budget spent", &shell, &repo)
20781            .await
20782            .expect("stop_reviewing retry");
20783        assert_eq!(
20784            runner.state.reviews.len(),
20785            1,
20786            "no new round was started: {:?}",
20787            runner.state.reviews
20788        );
20789        let last = runner.state.reviews.last().expect("round record");
20790        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20791        assert!(
20792            last.verified_at.expect("still known") > first_attempt_at,
20793            "a second reentry must be a fresh attempt, not a stale copy of the first"
20794        );
20795        assert_ne!(runner.state.status, RunStatus::Blocked);
20796
20797        held.release();
20798    }
20799
20800    #[cfg(unix)]
20801    #[tokio::test]
20802    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20803        ask_test_home();
20804        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20805            let tmp = tempfile::tempdir().unwrap();
20806            let repo = tmp.path().join("repo");
20807            std::fs::create_dir(&repo).unwrap();
20808            init_repo(&repo);
20809            let run_git = |args: &[&str]| {
20810                let out = std::process::Command::new("git")
20811                    .args(args)
20812                    .current_dir(&repo)
20813                    .output()
20814                    .unwrap();
20815                assert!(
20816                    out.status.success(),
20817                    "{args:?}: {}",
20818                    String::from_utf8_lossy(&out.stderr)
20819                );
20820            };
20821            let origin = tmp.path().join("origin.git");
20822            run_git(&["init", "--bare", origin.to_str().unwrap()]);
20823            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20824            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20825            run_git(&["add", "spec.rs"]);
20826            run_git(&["commit", "-m", "initial spec"]);
20827            run_git(&["push", "origin", "main"]);
20828            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20829            run_git(&["checkout", "-b", "fix/review"]);
20830            std::fs::write(
20831                repo.join("check.rs"),
20832                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20833            )
20834            .unwrap();
20835            run_git(&["add", "check.rs"]);
20836            run_git(&["commit", "-m", "use spec"]);
20837            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20838            run_git(&["checkout", "main"]);
20839            std::fs::write(
20840                repo.join("spec.rs"),
20841                "struct AgentSpec { max_turn_secs: u64 }\n",
20842            )
20843            .unwrap();
20844            run_git(&["add", "spec.rs"]);
20845            run_git(&["commit", "-m", "extend spec"]);
20846            run_git(&["push", "origin", "main"]);
20847            run_git(&["checkout", "fix/review"]);
20848
20849            let mut runner = runner_at(RunStatus::Reviewing);
20850            runner.state.repo = repo.clone();
20851            runner.state.base_commit = base;
20852            runner.state.pr_head = Some("fix/review".into());
20853            runner.state.config.graph.review_rounds = max_rounds;
20854            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20855            runner.state.config.graph.sessions = false;
20856            runner.state.config.disk.min_free_bytes = 0;
20857            runner.state.config.verify.e2e =
20858                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20859            runner.state.candidates = vec![Candidate {
20860                index: 0,
20861                label: 'A',
20862                agent: "fixture".into(),
20863                branch: "fix/review".into(),
20864                worktree: repo.clone(),
20865                summary: String::new(),
20866                stat: String::new(),
20867                files: 1,
20868                commits: 1,
20869                empty: false,
20870                failed: None,
20871                verified_noop: None,
20872                duration_ms: 0,
20873                folded: false,
20874            }];
20875            runner.state.tally = Some(Tally {
20876                first_choice: BTreeMap::new(),
20877                borda: BTreeMap::new(),
20878                winner: 'A',
20879                rankings: 0,
20880                unanimous_initial: true,
20881                deliberated: false,
20882                changed_votes: 0,
20883                unanimous_final: true,
20884                tie_break: None,
20885                judges: 0,
20886                present: 0,
20887                quorum: 0,
20888                met_quorum: true,
20889                uncontested: Some("review-only".into()),
20890            });
20891            let mut last = review_round(false, 0, 1, 1, false, false);
20892            last.head = reviewed.clone();
20893            last.e2e_deferred = true;
20894            last.e2e[0].resource_blocked = true;
20895            last.e2e[0].output_tail = "waiting for the shared build cache".into();
20896            if max_rounds == 1 {
20897                last.e2e.clear();
20898            }
20899            runner.state.reviews = vec![last];
20900            let calls = tmp.path().join("fix-calls");
20901            let mut fixer = spec("repair");
20902            fixer
20903                .env
20904                .insert("FIX_CALLS".into(), calls.display().to_string());
20905            fixer.command = vec!["sh".into(), "-c".into(), r#"
20906                echo fix >> "$FIX_CALLS"
20907                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20908                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20909                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20910                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20911            "#.into()];
20912            let mut reviewer = spec("reviewer");
20913            reviewer.command = vec![
20914                "sh".into(),
20915                "-c".into(),
20916                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20917                    .into(),
20918            ];
20919            runner.roles.fixer = Some(vec![fixer]);
20920            runner.roles.reviewers = vec![reviewer];
20921
20922            runner.sync_to_base().await.unwrap();
20923            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20924            assert_ne!(rebased, reviewed);
20925            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20926            if reentry {
20927                // Simulate a crash after catch-up evidence was saved but before
20928                // dispatching the fixer. No dedicated pending-repair field exists.
20929                let shell = runner.state.config.shell();
20930                runner
20931                    .stop_unresolved_review("deferred verification", &shell, &repo)
20932                    .await
20933                    .unwrap();
20934                assert!(runner.catchup_repairable());
20935                assert!(!calls.exists());
20936                runner.state = RunState::load(&runner.state.id).unwrap();
20937                assert!(runner.catchup_repairable());
20938            }
20939            runner.review_loop().await.unwrap();
20940            let first = &runner.state.reviews[0];
20941            assert_eq!(first.head, reviewed);
20942            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20943            assert!(
20944                first.verify_retried,
20945                "a reproduced build failure gets exactly one retry: {:?}",
20946                first.e2e
20947            );
20948            assert!(first.e2e[0].output_tail.contains("E0063"));
20949            assert!(!runner.catchup_repairable());
20950            if max_rounds == 2 {
20951                assert_eq!(
20952                    runner.state.status,
20953                    RunStatus::Gating,
20954                    "{}",
20955                    crate::report::run(&runner.state)
20956                );
20957                assert_eq!(runner.state.reviews.len(), 2);
20958                assert!(first.fix.as_ref().unwrap().committed);
20959                assert!(runner.state.reviews[1].clean);
20960                assert_eq!(
20961                    runner
20962                        .state
20963                        .reviews
20964                        .iter()
20965                        .filter(|r| r.fix.is_some())
20966                        .count(),
20967                    1
20968                );
20969                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20970                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20971            } else {
20972                assert_eq!(runner.state.status, RunStatus::Blocked);
20973                assert!(first.fix.is_none());
20974                assert!(!calls.exists());
20975                let recovery = crate::recovery::assess(
20976                    "prior",
20977                    Some(&runner.state),
20978                    &runner.landing_base(),
20979                    crate::recovery::Relation::Diverged,
20980                );
20981                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
20982                let reasons = recovery.reasons.join("\n");
20983                assert!(
20984                    reasons.contains(&reviewed) && reasons.contains(&rebased),
20985                    "{reasons}"
20986                );
20987                assert!(
20988                    reasons.contains("base sync to")
20989                        && reasons.contains("additional review is required"),
20990                    "{reasons}"
20991                );
20992            }
20993        }
20994    }
20995
20996    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
20997    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
20998    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
20999    /// own top-of-function fast path (`review_conclusion`) correctly reads
21000    /// this shape as `None` rather than guessing `Blocked`, and the loop's
21001    /// own `for` range is empty once the round budget is spent, so
21002    /// `review_loop` must retry the check itself rather than silently doing
21003    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
21004    /// above exercises directly, but through `review_loop`'s own entry point
21005    /// this time, proving the wiring between the two rather than just the
21006    /// retry logic in isolation.
21007    #[tokio::test]
21008    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21009        crate::run::pin_test_home();
21010        let home = crate::run::home();
21011
21012        let tmp = tempfile::tempdir().expect("tempdir");
21013        let repo = tmp.path().join("repo");
21014        std::fs::create_dir_all(&repo).unwrap();
21015        init_repo(&repo);
21016        let head = crate::git::rev_parse(&repo, "HEAD")
21017            .await
21018            .expect("rev-parse");
21019        let cache_dir = tmp.path().join("target");
21020
21021        let mut config = Config::default();
21022        config.verify.e2e = vec![format!(
21023            "CARGO_TARGET_DIR='{}' test -f README.md",
21024            cache_dir.display()
21025        )];
21026        config.graph.review_rounds = 1;
21027        config.graph.timeout_verify = Some(2);
21028        config.graph.timeout_resource_wait = 2;
21029
21030        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21031        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21032            .expect("no io error acquiring directly")
21033        {
21034            crate::cache::AcquireOutcome::Acquired(g) => g,
21035            crate::cache::AcquireOutcome::Busy(b) => {
21036                panic!("expected the direct acquire to win the lease first: {b:?}")
21037            }
21038        };
21039
21040        let mut state = RunState::new(
21041            repo.clone(),
21042            "main".to_owned(),
21043            head.clone(),
21044            "task".to_owned(),
21045            config,
21046        );
21047        state.candidates = vec![Candidate {
21048            index: 0,
21049            label: 'A',
21050            agent: "alpha".to_owned(),
21051            branch: "does-not-exist".to_owned(),
21052            worktree: repo.clone(),
21053            summary: String::new(),
21054            stat: String::new(),
21055            files: 0,
21056            commits: 0,
21057            empty: false,
21058            failed: None,
21059            verified_noop: None,
21060            duration_ms: 0,
21061            folded: false,
21062        }];
21063        state.tally = Some(Tally {
21064            first_choice: BTreeMap::from([('A', 1)]),
21065            borda: BTreeMap::new(),
21066            winner: 'A',
21067            rankings: 1,
21068            unanimous_initial: true,
21069            deliberated: false,
21070            changed_votes: 0,
21071            unanimous_final: true,
21072            tie_break: None,
21073            judges: 0,
21074            present: 0,
21075            quorum: 0,
21076            met_quorum: true,
21077            uncontested: Some("only candidate A produced a change".to_owned()),
21078        });
21079        // The exact shape a prior process's `stop_reviewing` would have left
21080        // on disk: the round budget's last round, a real attempt already
21081        // made and already resource-blocked.
21082        state.reviews = vec![ReviewRound {
21083            round: 1,
21084            head: head.clone(),
21085            verified_head: Some(head.clone()),
21086            verified_at: Some(jiff::Timestamp::now()),
21087            reviews: Vec::new(),
21088            e2e: vec![CommandOutcome {
21089                command: format!(
21090                    "CARGO_TARGET_DIR='{}' test -f README.md",
21091                    cache_dir.display()
21092                ),
21093                code: None,
21094                output_tail: "waiting for the shared build cache".to_owned(),
21095                duration_ms: 0,
21096                resource_blocked: true,
21097            }],
21098            fix: None,
21099            blocking: 1,
21100            answered: 1,
21101            expected: 1,
21102            clean: false,
21103            verify_retried: false,
21104            e2e_deferred: false,
21105            e2e_defer_reason: None,
21106            progressed: false,
21107            vote_split: false,
21108            reconsideration: Vec::new(),
21109            verdict: None,
21110        }];
21111
21112        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
21113        let mut runner = Runner {
21114            state,
21115            roles: ResolvedRoles {
21116                implementers: Vec::new(),
21117                judges: Vec::new(),
21118                reviewers: Vec::new(),
21119                fixer: None,
21120                conductor: conductor(),
21121                implementer_roster: Vec::new(),
21122                judge_roster: Vec::new(),
21123                reviewer_roster: Vec::new(),
21124            },
21125            sem: Arc::new(Semaphore::new(1)),
21126            pause: Pause::new(),
21127            interrupt: Pause::new(),
21128        };
21129
21130        // The lease is still held throughout, so this reentry's own retry is
21131        // also contended — proving `review_loop` actually tried again (not
21132        // that it happened to succeed) is what the timestamp comparison
21133        // below is for.
21134        runner.review_loop().await.expect("review_loop");
21135
21136        assert_eq!(
21137            runner.state.reviews.len(),
21138            1,
21139            "no new round was started on top of the unresolved one: {:?}",
21140            runner.state.reviews
21141        );
21142        let last = &runner.state.reviews[0];
21143        assert_eq!(
21144            last.e2e_status(),
21145            E2eStatus::ResourceBlocked,
21146            "still contended: {last:?}"
21147        );
21148        assert!(
21149            last.verified_at.expect("still known") > first_attempt_at,
21150            "review_loop must have actually retried the check, not left it exactly as found"
21151        );
21152        assert_ne!(
21153            runner.state.status,
21154            RunStatus::Blocked,
21155            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21156            runner.state.status
21157        );
21158
21159        held.release();
21160    }
21161
21162    #[tokio::test]
21163    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21164        crate::run::pin_test_home();
21165        for status in [RunStatus::Blocked, RunStatus::Gating] {
21166            let mut runner = runner_at(status);
21167            runner.state.config.graph.land = true;
21168            runner.state.gate_ran = true;
21169            runner.state.merge = Some(MergeOutcome {
21170                mode: MergeMode::Pr,
21171                ok: false,
21172                detail: "the fixer produced no commit".into(),
21173                empty: false,
21174            });
21175            let url = "https://example.invalid/o/r/pull/1";
21176            runner.state.pr = Some(crate::run::PrRecord {
21177                url: url.into(),
21178                number: 1,
21179                state: "open".into(),
21180                checks: "red".into(),
21181                round: 1,
21182                rounds: 1,
21183                red_at_merge: Vec::new(),
21184            });
21185            runner.state.save().unwrap();
21186            // The nonexistent repository makes observation fail before a forge
21187            // command can run. We still must reach land, rather than silently
21188            // return from the graph with its old gate and merge records.
21189            runner.execute_graph().await.unwrap();
21190            assert_eq!(runner.state.status, RunStatus::Blocked);
21191            assert!(runner.state.land_sync_pending);
21192            assert!(runner.state.events.iter().any(|event|
21193                event.node == "land" && event.message == format!("watching {url}")));
21194            assert_eq!(
21195                runner.state.merge.as_ref().unwrap().detail,
21196                "the fixer produced no commit"
21197            );
21198        }
21199    }
21200
21201    #[tokio::test]
21202    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21203        crate::run::pin_test_home();
21204        let tmp = tempfile::tempdir().expect("tempdir");
21205        let repo = tmp.path().join("repo");
21206        std::fs::create_dir_all(&repo).unwrap();
21207        init_repo(&repo);
21208
21209        let mut config = Config::default();
21210        config.merge.mode = MergeMode::Pr;
21211        config.graph.land = true;
21212        config.graph.land_approval = false;
21213
21214        let mut state = RunState::new(
21215            repo.clone(),
21216            "main".to_owned(),
21217            "deadbeef".to_owned(),
21218            "task".to_owned(),
21219            config,
21220        );
21221        state.candidates = vec![Candidate {
21222            index: 0,
21223            label: 'A',
21224            agent: "alpha".to_owned(),
21225            branch: "does-not-exist".to_owned(),
21226            worktree: repo.clone(),
21227            summary: String::new(),
21228            stat: String::new(),
21229            files: 0,
21230            commits: 0,
21231            empty: false,
21232            failed: None,
21233            verified_noop: None,
21234            duration_ms: 0,
21235            folded: false,
21236        }];
21237        state.tally = Some(Tally {
21238            first_choice: BTreeMap::from([('A', 1)]),
21239            borda: BTreeMap::new(),
21240            winner: 'A',
21241            rankings: 1,
21242            unanimous_initial: true,
21243            deliberated: false,
21244            changed_votes: 0,
21245            unanimous_final: true,
21246            tie_break: None,
21247            judges: 0,
21248            present: 0,
21249            quorum: 0,
21250            met_quorum: true,
21251            uncontested: Some("only candidate A produced a change".to_owned()),
21252        });
21253        state.reviews = vec![ReviewRound {
21254            round: 1,
21255            head: "deadbeef".to_owned(),
21256            verified_head: None,
21257            verified_at: None,
21258            reviews: Vec::new(),
21259            e2e: Vec::new(),
21260            fix: None,
21261            blocking: 0,
21262            answered: 0,
21263            expected: 0,
21264            clean: true,
21265            verify_retried: false,
21266            e2e_deferred: false,
21267            e2e_defer_reason: None,
21268            progressed: false,
21269            vote_split: false,
21270            reconsideration: Vec::new(),
21271            verdict: None,
21272        }];
21273        state.gate = vec![CommandOutcome {
21274            command: "test".to_owned(),
21275            code: Some(0),
21276            output_tail: String::new(),
21277            duration_ms: 0,
21278            resource_blocked: false,
21279        }];
21280        state.gate_ran = true;
21281        // A first pass through `merge` already pushed and opened this pull
21282        // request; `status` is `Landing` because a previous call into `land`
21283        // parked or was interrupted before it reached a terminal outcome.
21284        state.status = RunStatus::Landing;
21285        state.merge = Some(MergeOutcome {
21286            mode: MergeMode::Pr,
21287            ok: true,
21288            detail: "https://example.invalid/x/y/pull/1".to_owned(),
21289            empty: false,
21290        });
21291
21292        // The Landing-resume shortcut calls `run_land` directly rather than
21293        // through `merge`, which is exactly the call site that used to skip
21294        // `settle_questions` - see the fixture below.
21295        ask_test_home();
21296        // A real resume loads a persisted run; driver ownership is checked
21297        // against that record before the Landing shortcut is entered.
21298        state.save().expect("save landing run");
21299        let store = ask::Questions::open();
21300        let q = ask_open_question(&store, &state.id);
21301
21302        let mut runner = Runner {
21303            state,
21304            roles: ResolvedRoles {
21305                implementers: Vec::new(),
21306                judges: Vec::new(),
21307                reviewers: Vec::new(),
21308                fixer: None,
21309                conductor: conductor(),
21310                implementer_roster: Vec::new(),
21311                judge_roster: Vec::new(),
21312                reviewer_roster: Vec::new(),
21313            },
21314            sem: Arc::new(Semaphore::new(1)),
21315            pause: Pause::new(),
21316            interrupt: Pause::new(),
21317        };
21318
21319        // `execute`, not `merge` directly: the Landing-resume shortcut lives
21320        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
21321        // exactly because `review_loop` would otherwise clobber the marker
21322        // first.
21323        runner.execute().await.expect("execute");
21324
21325        assert_eq!(
21326            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21327            Some("https://example.invalid/x/y/pull/1"),
21328            "reentry must not push again or open a second pull request over the \
21329             one `land` is already watching"
21330        );
21331        assert_ne!(
21332            runner.state.status,
21333            RunStatus::Landing,
21334            "land could not actually reach the fake pull request, so it must \
21335             have given up rather than left the run silently parked forever"
21336        );
21337        // `land` could not reach the fake pull request, so it gave up into
21338        // `Blocked` - still resumable, so the question must not have been
21339        // swept just because this branch now also calls `settle_questions`.
21340        assert_eq!(runner.state.status, RunStatus::Blocked);
21341        assert!(
21342            store.get(&q.id).unwrap().status.open(),
21343            "Blocked is still alive; settle_questions must have been a no-op here"
21344        );
21345    }
21346
21347    fn state_with_round(round: ReviewRound) -> RunState {
21348        let mut s = RunState::new(
21349            PathBuf::from("/repo"),
21350            "main".to_owned(),
21351            "abc1234".to_owned(),
21352            "add retries".to_owned(),
21353            Config::default(),
21354        );
21355        s.reviews = vec![round];
21356        s
21357    }
21358
21359    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21360        crate::verdict::Finding {
21361            id: id.to_owned(),
21362            severity,
21363            file: None,
21364            line: None,
21365            title: title.to_owned(),
21366            detail: String::new(),
21367        }
21368    }
21369
21370    #[test]
21371    fn pr_body_names_open_findings_and_declined_ones() {
21372        let round = ReviewRound {
21373            round: 2,
21374            head: "deadbee".to_owned(),
21375            verified_head: None,
21376            verified_at: None,
21377            reviews: vec![ReviewRecord {
21378                attempts: 0,
21379                reviewer: 1,
21380                agent: "alpha".to_owned(),
21381                summary: String::new(),
21382                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21383                vote: None,
21384                failed: None,
21385                duration_ms: 0,
21386            }],
21387            e2e: vec![CommandOutcome {
21388                command: "cargo test".to_owned(),
21389                code: Some(0),
21390                output_tail: String::new(),
21391                duration_ms: 0,
21392                resource_blocked: false,
21393            }],
21394            verify_retried: false,
21395            e2e_deferred: false,
21396            e2e_defer_reason: None,
21397            fix: Some(FixRecord {
21398                agent: "alpha".to_owned(),
21399                addressed: Vec::new(),
21400                rejected: vec![crate::verdict::Rejection {
21401                    id: "R1-1-1".to_owned(),
21402                    why: "not reachable from any caller".to_owned(),
21403                }],
21404                notes: String::new(),
21405                committed: true,
21406                failed: None,
21407                duration_ms: 0,
21408                continuation: None,
21409            }),
21410            blocking: 0,
21411            answered: 1,
21412            expected: 1,
21413            clean: false,
21414            progressed: true,
21415            vote_split: false,
21416            reconsideration: Vec::new(),
21417            verdict: None,
21418        };
21419        let state = state_with_round(round);
21420        let body = pr_message(&state, 'A').body;
21421
21422        assert!(body.contains("add retries"), "the task must still be there");
21423        assert!(body.contains("R2-1-1"), "{body}");
21424        assert!(!body.contains("unused import"), "{body}");
21425        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21426        assert!(
21427            !body.contains("not reachable from any caller"),
21428            "the reason it was declined: {body}"
21429        );
21430    }
21431
21432    #[test]
21433    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21434        let round = ReviewRound {
21435            round: 1,
21436            head: "deadbee".to_owned(),
21437            verified_head: None,
21438            verified_at: None,
21439            reviews: vec![ReviewRecord {
21440                attempts: 0,
21441                reviewer: 1,
21442                agent: "alpha".to_owned(),
21443                summary: String::new(),
21444                findings: Vec::new(),
21445                vote: None,
21446                failed: None,
21447                duration_ms: 0,
21448            }],
21449            e2e: Vec::new(),
21450            verify_retried: false,
21451            e2e_deferred: false,
21452            e2e_defer_reason: None,
21453            fix: None,
21454            blocking: 0,
21455            answered: 1,
21456            expected: 1,
21457            clean: true,
21458            progressed: false,
21459            vote_split: false,
21460            reconsideration: Vec::new(),
21461            verdict: None,
21462        };
21463        let state = state_with_round(round);
21464        let body = pr_message(&state, 'A').body;
21465        assert!(!body.contains("Open review findings"), "{body}");
21466        assert!(!body.contains("Declined"), "{body}");
21467    }
21468
21469    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21470        let mut state = RunState::new(
21471            PathBuf::from("/repo"),
21472            "main".to_owned(),
21473            "abc1234".to_owned(),
21474            instruction.to_owned(),
21475            Config::default(),
21476        );
21477        // These fixtures test execution flow, independent of host disk capacity.
21478        state.config.disk.min_free_bytes = 0;
21479        state.candidates.push(Candidate {
21480            index: 0,
21481            label: 'A',
21482            agent: "alpha".to_owned(),
21483            branch: "magi/x/A".to_owned(),
21484            worktree: PathBuf::from("/wt"),
21485            summary: summary.to_owned(),
21486            stat: String::new(),
21487            files: 1,
21488            commits: 1,
21489            empty: false,
21490            failed: None,
21491            verified_noop: None,
21492            folded: false,
21493            duration_ms: 0,
21494        });
21495        state
21496    }
21497
21498    fn review_state(subjects: &[&str]) -> RunState {
21499        let mut state = state_with_summary(
21500            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21501            "",
21502        );
21503        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21504        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21505        state
21506    }
21507
21508    /// A branch rebased onto a main that moved past the recorded
21509    /// `base_commit` is titled from its own first commit, never main's.
21510    #[tokio::test]
21511    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21512        ask_test_home();
21513        let tmp = tempfile::tempdir().unwrap();
21514        let repo = tmp.path().join("repo");
21515        std::fs::create_dir_all(&repo).unwrap();
21516        init_repo(&repo);
21517        let origin = tmp.path().join("origin.git");
21518        let g = |dir: &Path, args: &[&str]| {
21519            let out = std::process::Command::new("git")
21520                .args(args)
21521                .current_dir(dir)
21522                .quiet()
21523                .output()
21524                .expect("spawn git");
21525            assert!(
21526                out.status.success(),
21527                "git {args:?}: {}",
21528                String::from_utf8_lossy(&out.stderr)
21529            );
21530        };
21531        g(
21532            tmp.path(),
21533            &[
21534                "clone",
21535                "--bare",
21536                "-q",
21537                repo.to_str().unwrap(),
21538                origin.to_str().unwrap(),
21539            ],
21540        );
21541        g(
21542            &repo,
21543            &["remote", "add", "origin", origin.to_str().unwrap()],
21544        );
21545        let c1 = git::rev_parse(&repo, "main").await.unwrap();
21546
21547        // Main moves on; the branch is built on top of the new main.
21548        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21549        g(&repo, &["add", "-A"]);
21550        g(
21551            &repo,
21552            &["commit", "-q", "-m", "chore(deps): update a crate"],
21553        );
21554        g(&repo, &["push", "-q", "origin", "main"]);
21555        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21556        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21557        g(&repo, &["add", "-A"]);
21558        g(
21559            &repo,
21560            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21561        );
21562        g(&repo, &["checkout", "-q", "main"]);
21563
21564        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21565        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21566        // Without a readable tracking ref the recorded base's merge base is used.
21567        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21568        assert_eq!(fallback, c1);
21569
21570        let mut state = review_state(&[
21571            "chore(deps): update a crate",
21572            "fix(daemon): apply a chosen action",
21573        ]);
21574        state.repo = repo.clone();
21575        state.base_branch = "main".to_owned();
21576        state.base_commit = c1;
21577        refresh_reviewed_commits(&mut state, "feat/own").await;
21578        assert_eq!(
21579            state.reviewed_commits,
21580            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21581        );
21582        assert_eq!(
21583            review_title(&state).as_deref(),
21584            Some("fix(daemon): apply a chosen action")
21585        );
21586        assert_eq!(
21587            leaked_subjects(&state, "feat/own").await,
21588            Some(vec!["chore(deps): update a crate".to_owned()])
21589        );
21590        // A stale tracking ref is still used when the fetch fails, but the
21591        // leak list is withheld.
21592        state.config.merge.remote = "nowhere".to_owned();
21593        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21594    }
21595
21596    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
21597    /// with `origin/main` one commit behind it, and a summarizing agent that
21598    /// prints whatever `reply` holds.
21599    struct SummaryFixture {
21600        _tmp: tempfile::TempDir,
21601        repo: PathBuf,
21602        reply: PathBuf,
21603        runner: Runner,
21604    }
21605
21606    fn summary_fixture() -> SummaryFixture {
21607        ask_test_home();
21608        let tmp = tempfile::tempdir().unwrap();
21609        let repo = tmp.path().join("repo");
21610        std::fs::create_dir_all(&repo).unwrap();
21611        init_repo(&repo);
21612        let origin = tmp.path().join("origin.git");
21613        let g = |args: &[&str]| {
21614            let out = std::process::Command::new("git")
21615                .args(args)
21616                .current_dir(&repo)
21617                .quiet()
21618                .output()
21619                .expect("spawn git");
21620            assert!(
21621                out.status.success(),
21622                "git {args:?}: {}",
21623                String::from_utf8_lossy(&out.stderr)
21624            );
21625        };
21626        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21627        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21628        g(&["push", "-q", "origin", "main"]);
21629        g(&["checkout", "-q", "-b", "feat/own"]);
21630        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21631        g(&["add", "-A"]);
21632        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21633        let reply = tmp.path().join("reply.txt");
21634        std::fs::write(
21635            &reply,
21636            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21637        )
21638        .unwrap();
21639        let mut runner = runner_at(RunStatus::Gating);
21640        let mut state = review_state(&["feat(own): add the own file"]);
21641        state.repo = repo.clone();
21642        state.base_branch = "main".to_owned();
21643        state.candidates[0].branch = "feat/own".to_owned();
21644        state.candidates[0].worktree = repo.clone();
21645        let mut author = spec("alpha");
21646        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21647        author
21648            .env
21649            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21650        state.config.agents = vec![author];
21651        runner.state = state;
21652        SummaryFixture {
21653            _tmp: tmp,
21654            repo,
21655            reply,
21656            runner,
21657        }
21658    }
21659
21660    async fn posted(f: &mut SummaryFixture) -> PrMessage {
21661        let winner = f.runner.state.candidates[0].clone();
21662        f.runner
21663            .guarded_pr_message(&winner, None, true)
21664            .await
21665            .unwrap()
21666            .expect("an English description never parks")
21667    }
21668
21669    fn clean_round_with_minor_findings() -> ReviewRound {
21670        let mut round = passing_round(2, None);
21671        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21672        round.reviews = vec![ReviewRecord {
21673            attempts: 0,
21674            reviewer: 1,
21675            agent: "alpha".into(),
21676            summary: "内部レビュー".into(),
21677            findings: vec![
21678                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21679                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21680            ],
21681            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21682            failed: None,
21683            duration_ms: 0,
21684        }];
21685        round
21686    }
21687
21688    #[tokio::test]
21689    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21690        for review_only in [false, true] {
21691            let mut f = summary_fixture();
21692            if !review_only {
21693                f.runner.state.candidates[0].agent = "alpha".into();
21694            }
21695            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21696            assert!(f.runner.state.reviews[0].clean);
21697            assert!(f.runner.state.reviews[0].fix.is_none());
21698            assert!(f.runner.state.open_findings().is_empty());
21699            assert_eq!(
21700                crate::pr_outcome::required_finding_ids(&f.runner.state),
21701                ["R2-1-1", "R2-1-2"]
21702            );
21703            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21704            let evidence = crate::pr_outcome::summary_evidence(
21705                &f.runner.state,
21706                crate::pr_outcome::FinalHead::At(&head),
21707            );
21708            assert!(
21709                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21710            );
21711            // The ordinary reply omits both IDs: no cached description is adopted.
21712            let omitted = posted(&mut f).await;
21713            assert!(omitted.body.contains("No description"));
21714            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21715            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21716            assert!(f.runner.state.change_summary.is_none());
21717            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21718            let described = posted(&mut f).await;
21719            assert!(described.body.contains("improve documentation for callers"));
21720            assert!(described.body.contains("remaining edge case"));
21721            assert!(described.body.contains("## Open review findings"));
21722            assert!(!described.body.contains("未修正"));
21723        }
21724    }
21725
21726    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21727        let mut f = summary_fixture();
21728        if !review_only {
21729            f.runner.state.candidates[0].agent = "alpha".into();
21730        }
21731        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21732        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21733        fixed
21734            .fix
21735            .as_mut()
21736            .unwrap()
21737            .rejected
21738            .push(crate::verdict::Rejection {
21739                id: "R1-1-1".into(),
21740                why: "呼び出されないコードのため辞退".into(),
21741            });
21742        let mut clean = clean_round_with_minor_findings();
21743        clean.verified_head = Some(head.clone());
21744        f.runner.state.reviews = vec![fixed, clean];
21745        ok_gate(&mut f.runner.state, Some(&head));
21746        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21747        f.runner.state.config.agents.push(scripted("beta", reply));
21748        f.runner.state.config.roles.language_judge =
21749            Some(crate::config::AgentChoice::One("beta".into()));
21750        f
21751    }
21752
21753    #[tokio::test]
21754    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21755        for review_only in [false, true] {
21756            let mut f = rewrite_fixture_with_findings(review_only,
21757                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21758            let m = posted(&mut f).await;
21759            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21760            let expected =
21761                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21762            assert!(m.body.contains(&expected));
21763            assert!(m.body.contains("## Review rounds"));
21764            assert!(
21765                m.body
21766                    .contains("Gate (`verify.gate`): passed on the final commit")
21767            );
21768            assert!(
21769                m.body
21770                    .contains("Tests (`verify.e2e`): passed on the final commit")
21771            );
21772            let c = f.runner.state.change_summary.as_ref().unwrap();
21773            assert!(
21774                m.body
21775                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21776            );
21777            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21778            assert!(crate::pr_outcome::covers_required_findings(
21779                &f.runner.state,
21780                &m.body
21781            ));
21782            assert!(m.body.contains("documentation remains incomplete"));
21783            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21784            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21785            assert!(f.runner.state.github_text.is_none());
21786        }
21787    }
21788
21789    #[tokio::test]
21790    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21791        for review_only in [false, true] {
21792            for reply in [
21793                "TITLE: fix: public description\nA short English description.",
21794                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21795            ] {
21796                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21797                let winner = f.runner.state.candidates[0].clone();
21798                let posted = f
21799                    .runner
21800                    .guarded_pr_message(&winner, None, true)
21801                    .await
21802                    .unwrap();
21803                assert!(posted.is_none());
21804                assert!(f.runner.state.parked);
21805                let withheld = f.runner.state.github_text.as_ref().unwrap();
21806                assert!(
21807                    withheld.rewrite.is_none(),
21808                    "an incomplete rewrite must not be selectable"
21809                );
21810                let q = ask::Questions::open()
21811                    .get(withheld.question.as_ref().unwrap())
21812                    .unwrap();
21813                assert!(
21814                    !q.choices
21815                        .contains(&crate::github_text::USE_REWRITE.to_owned())
21816                );
21817                assert!(
21818                    q.detail.contains("R1-1-1")
21819                        && q.detail.contains("R2-1-1")
21820                        && q.detail.contains("R2-1-2")
21821                );
21822            }
21823        }
21824    }
21825
21826    #[tokio::test]
21827    async fn japanese_reports_stay_local_in_both_public_run_modes() {
21828        for review_only in [false, true] {
21829            let mut f = summary_fixture();
21830            if !review_only {
21831                f.runner.state.candidates[0].agent = "alpha".into();
21832            }
21833            f.runner.state.instruction = "内部タスクの説明".into();
21834            f.runner.state.candidates[0].summary = "内部の候補報告".into();
21835            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21836            f.runner.state.reviews = vec![
21837                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21838                round_with_notes(2, Some("二回目の内部修正報告です。")),
21839                passing_round(3, Some(&head)),
21840            ];
21841            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21842            ok_gate(&mut f.runner.state, Some(&head));
21843            let captured = f.repo.join("public-prompt.txt");
21844            let author = &mut f.runner.state.config.agents[0];
21845            author.command = vec![
21846                "sh".into(),
21847                "-c".into(),
21848                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21849            ];
21850            author.env.insert(
21851                "PROMPT_FILE".into(),
21852                captured.to_string_lossy().into_owned(),
21853            );
21854            let m = posted(&mut f).await;
21855            assert!(m.body.contains("Adds `own.txt`"));
21856            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21857            assert!(m.body.contains("R1-1-1"));
21858            assert!(
21859                m.body
21860                    .contains("Gate (`verify.gate`): passed on the final commit")
21861            );
21862            assert!(
21863                m.body
21864                    .contains("Tests (`verify.e2e`): passed on the final commit")
21865            );
21866            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21867            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21868            let prompt = std::fs::read_to_string(captured).unwrap();
21869            assert!(prompt.contains(&format!(
21870                "git diff {} {head}",
21871                f.runner.state.change_summary.as_ref().unwrap().base
21872            )));
21873            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21874            assert_eq!(
21875                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21876                "二回目の内部修正報告です。"
21877            );
21878        }
21879    }
21880
21881    #[tokio::test]
21882    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21883        for review_only in [false, true] {
21884            let mut f = summary_fixture();
21885            if !review_only {
21886                f.runner.state.candidates[0].agent = "alpha".into();
21887            }
21888            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21889            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21890            std::fs::write(&f.reply, "no usable description").unwrap();
21891            let failed = posted(&mut f).await;
21892            assert!(
21893                failed
21894                    .body
21895                    .contains("No description of what this change does could be generated")
21896            );
21897            assert!(failed.body.is_ascii());
21898            assert!(f.runner.state.github_text.is_none());
21899            assert!(f.runner.state.change_summary.is_none());
21900            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21901            let evidence = crate::pr_outcome::summary_evidence(
21902                &f.runner.state,
21903                crate::pr_outcome::FinalHead::At(&head),
21904            );
21905            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21906            assert!(
21907                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21908                    .exists()
21909            );
21910            f.runner.state =
21911                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21912            std::fs::write(
21913                &f.reply,
21914                "TITLE: feat(own): add the own file\nAdds the committed own file.",
21915            )
21916            .unwrap();
21917            let retried = posted(&mut f).await;
21918            assert!(retried.body.contains("Adds the committed own file."));
21919            assert_eq!(
21920                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21921                "日本語の修正報告"
21922            );
21923        }
21924    }
21925
21926    #[tokio::test]
21927    async fn same_head_verification_and_findings_invalidate_public_prose() {
21928        let mut f = summary_fixture();
21929        posted(&mut f).await;
21930        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21931        let original = f
21932            .runner
21933            .state
21934            .change_summary
21935            .as_ref()
21936            .unwrap()
21937            .evidence
21938            .clone();
21939        ok_gate(&mut f.runner.state, Some(&head));
21940        std::fs::write(
21941            &f.reply,
21942            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21943        )
21944        .unwrap();
21945        let verified = posted(&mut f).await;
21946        assert!(verified.body.contains("now verified"));
21947        assert_ne!(
21948            f.runner.state.change_summary.as_ref().unwrap().evidence,
21949            original
21950        );
21951        let mut round = round_with_notes(1, Some("内部報告"));
21952        round.reviews.push(ReviewRecord {
21953            attempts: 0,
21954            reviewer: 1,
21955            agent: "alpha".into(),
21956            summary: "内部レビュー".into(),
21957            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21958            vote: None,
21959            failed: None,
21960            duration_ms: 0,
21961        });
21962        round
21963            .fix
21964            .as_mut()
21965            .unwrap()
21966            .rejected
21967            .push(crate::verdict::Rejection {
21968                id: "R1-1-1".into(),
21969                why: "呼び出し元から到達できない".into(),
21970            });
21971        f.runner.state.reviews = vec![round];
21972        // A description omitting the required ID is unusable, even with a title.
21973        let omitted = posted(&mut f).await;
21974        assert!(omitted.body.contains("No description"));
21975        assert!(omitted.body.contains("R1-1-1"));
21976        assert!(!omitted.body.contains("到達"));
21977        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21978        let described = posted(&mut f).await;
21979        assert!(described.body.contains("unreachable from callers"));
21980        assert!(described.body.contains("R1-1-1"));
21981        assert!(!described.body.contains("到達"));
21982        let evidence = f
21983            .runner
21984            .state
21985            .change_summary
21986            .as_ref()
21987            .unwrap()
21988            .evidence
21989            .clone();
21990        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
21991        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
21992        assert!(posted(&mut f).await.body.contains("rationale updated"));
21993        assert_ne!(
21994            f.runner.state.change_summary.as_ref().unwrap().evidence,
21995            evidence
21996        );
21997        // Old serialized summaries have no evidence fingerprint and must regenerate.
21998        f.runner
21999            .state
22000            .change_summary
22001            .as_mut()
22002            .unwrap()
22003            .evidence
22004            .clear();
22005        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22006        assert!(posted(&mut f).await.body.contains("current specification"));
22007    }
22008
22009    #[tokio::test]
22010    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22011        let mut f = summary_fixture();
22012        let m = posted(&mut f).await;
22013        assert_eq!(m.title, "feat(own): add the own file");
22014        assert!(
22015            m.body
22016                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22017            "{}",
22018            m.body
22019        );
22020        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22021        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22022        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22023        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22024        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22025        let kept = f.runner.state.change_summary.as_ref().unwrap();
22026        assert_eq!(
22027            (kept.head.as_str(), kept.base.as_str()),
22028            (head.as_str(), base.as_str())
22029        );
22030        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22031    }
22032
22033    #[tokio::test]
22034    async fn review_only_summary_receives_the_configured_github_language() {
22035        let mut f = summary_fixture();
22036        f.runner.state.config.graph.github_language = "ja".into();
22037        let captured = f.repo.join("summary-prompt.txt");
22038        let author = &mut f.runner.state.config.agents[0];
22039        author.command = vec![
22040            "sh".into(),
22041            "-c".into(),
22042            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22043        ];
22044        author.env.insert(
22045            "PROMPT_FILE".into(),
22046            captured.to_string_lossy().into_owned(),
22047        );
22048        std::fs::write(
22049            &f.reply,
22050            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22051        )
22052        .unwrap();
22053        let m = posted(&mut f).await;
22054        let prompt = std::fs::read_to_string(captured).unwrap();
22055        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22056        assert!(!prompt.contains("in English"));
22057        assert_eq!(m.title, "feat(own): ファイルを追加");
22058        assert!(m.body.contains("own.txt を追加します。"));
22059    }
22060
22061    #[tokio::test]
22062    async fn review_fixes_come_after_the_whole_change_summary() {
22063        let mut f = summary_fixture();
22064        f.runner.state.reviews = vec![
22065            round_with_notes(1, Some("handled the PATH case")),
22066            round_with_notes(2, None),
22067        ];
22068        let m = posted(&mut f).await;
22069        let summary = m.body.find("## Summary").expect("summary");
22070        let fixes = m.body.find("## Review rounds").expect("review rounds");
22071        assert!(summary < fixes, "{}", m.body);
22072        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22073        assert!(m.body.contains("## Final verification"), "{}", m.body);
22074    }
22075
22076    #[tokio::test]
22077    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22078        let mut f = summary_fixture();
22079        let first = posted(&mut f).await;
22080        let first_head = f.runner.state.change_summary.clone().unwrap().head;
22081        // The same head reuses the record: a different reply is not read.
22082        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22083        let again = posted(&mut f).await;
22084        assert_eq!((again.title, again.body), (first.title, first.body));
22085
22086        let g = |args: &[&str]| {
22087            let out = std::process::Command::new("git")
22088                .args(args)
22089                .current_dir(&f.repo)
22090                .quiet()
22091                .output()
22092                .expect("spawn git");
22093            assert!(out.status.success(), "git {args:?}");
22094        };
22095        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22096        g(&["add", "-A"]);
22097        g(&["commit", "-q", "-m", "fix(own): handle more"]);
22098        let second = posted(&mut f).await;
22099        let kept = f.runner.state.change_summary.clone().unwrap();
22100        assert_ne!(kept.head, first_head);
22101        assert!(second.body.contains("Not read yet."), "{}", second.body);
22102        assert!(!second.body.contains("so the first summary is read"));
22103        assert_eq!(second.title, "feat(own): other");
22104    }
22105
22106    #[tokio::test]
22107    async fn an_unavailable_summary_is_stated_and_never_invented() {
22108        let mut f = summary_fixture();
22109        std::fs::write(&f.reply, "no title line here").unwrap();
22110        let m = posted(&mut f).await;
22111        assert!(
22112            m.body
22113                .contains("No description of what this change does could be generated"),
22114            "{}",
22115            m.body
22116        );
22117        assert!(m.body.contains("internal reports remain in local records"));
22118        assert!(f.runner.state.change_summary.is_none());
22119        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
22120        // A preview asks nobody and claims nothing.
22121        let winner = f.runner.state.candidates[0].clone();
22122        let preview = f
22123            .runner
22124            .guarded_pr_message(&winner, None, false)
22125            .await
22126            .unwrap()
22127            .unwrap();
22128        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
22129    }
22130
22131    #[test]
22132    fn pr_message_review_single_commit_uses_its_subject() {
22133        let state = review_state(&["feat(nats): per-role user"]);
22134        let m = pr_message(&state, 'A');
22135        assert_eq!(m.title, "feat(nats): per-role user");
22136        assert!(!m.body.contains("Review the work already"), "{}", m.body);
22137        assert!(m.body.contains("## Commits under review"), "{}", m.body);
22138    }
22139
22140    #[test]
22141    fn pr_message_review_multi_commit_takes_the_oldest() {
22142        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
22143        let m = pr_message(&state, 'A');
22144        assert_eq!(m.title, "feat: the change");
22145        for s in ["feat: the change", "fix: typo", "fix: again"] {
22146            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
22147        }
22148    }
22149
22150    #[test]
22151    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
22152        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
22153            let mut state = review_state(&[first, "fix: later fixup"]);
22154            state.candidates[0].branch = "機能/ログイン".to_owned();
22155            let m = pr_message(&state, 'A');
22156            assert!(
22157                m.title.starts_with("chore: land candidate A of run"),
22158                "{}",
22159                m.title
22160            );
22161        }
22162    }
22163
22164    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22165        BranchFacts {
22166            commits: commits
22167                .iter()
22168                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22169                .collect(),
22170            stat: stat.to_owned(),
22171        }
22172    }
22173
22174    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22175        let mut r = review_round(true, 0, 1, 1, true, true);
22176        r.round = round;
22177        r.fix = notes.map(|n| FixRecord {
22178            agent: "fixer".to_owned(),
22179            addressed: Vec::new(),
22180            rejected: Vec::new(),
22181            notes: n.to_owned(),
22182            committed: true,
22183            failed: None,
22184            duration_ms: 0,
22185            continuation: None,
22186        });
22187        r
22188    }
22189
22190    #[test]
22191    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22192        let state = review_state(&["ignored"]);
22193        let f = facts(
22194            &[
22195                (
22196                    "fix(login): resolve PATH on macOS",
22197                    "Login shells skip rc files.",
22198                ),
22199                ("fix: address review", ""),
22200            ],
22201            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22202        );
22203        let m = pr_message_with(&state, 'A', Some(&f));
22204        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22205        assert!(
22206            m.body
22207                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
22208            "{}",
22209            m.body
22210        );
22211        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22212        assert!(m.body.contains("## Diff stat"), "{}", m.body);
22213        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22214        for banned in [
22215            "Review the work already",
22216            "no task statement",
22217            "Original task",
22218        ] {
22219            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22220        }
22221        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22222    }
22223
22224    #[test]
22225    fn pr_message_review_truncates_a_huge_first_commit_body() {
22226        let state = review_state(&["ignored"]);
22227        let f = facts(
22228            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22229            "s",
22230        );
22231        let m = pr_message_with(&state, 'A', Some(&f));
22232        assert!(m.body.len() < 30_000, "{}", m.body.len());
22233        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22234        assert!(
22235            m.body.contains("- ... 1 more commit(s)"),
22236            "{}",
22237            m.body.len()
22238        );
22239        assert!(m.body.ends_with("magi:candidate-a\n"));
22240    }
22241
22242    #[test]
22243    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22244        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22245        assert_eq!(m.title, "feat: x");
22246        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22247        assert!(!m.body.contains("Diff stat"), "{}", m.body);
22248        assert!(!m.body.contains("no task statement"), "{}", m.body);
22249    }
22250
22251    #[test]
22252    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22253        let mut state = review_state(&["日本語の件名"]);
22254        state.candidates[0].branch = "fix/macos-login-path".to_owned();
22255        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22256        state.candidates[0].branch = "機能/ログイン".to_owned();
22257        assert!(
22258            pr_message(&state, 'A')
22259                .title
22260                .starts_with("chore: land candidate A")
22261        );
22262    }
22263
22264    #[test]
22265    fn pr_message_review_fixes_survive_a_clean_final_round() {
22266        let mut state = review_state(&["feat: x"]);
22267        state.reviews = vec![
22268            round_with_notes(1, Some("handled the PATH case")),
22269            round_with_notes(2, None),
22270        ];
22271        let body = pr_message(&state, 'A').body;
22272        assert!(
22273            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22274            "{body}"
22275        );
22276        assert!(body.contains("### Round 1"), "{body}");
22277
22278        state.reviews = vec![
22279            round_with_notes(1, Some("first fix")),
22280            round_with_notes(2, Some("")),
22281            round_with_notes(3, Some("second fix")),
22282            round_with_notes(4, None),
22283        ];
22284        let body = pr_message(&state, 'A').body;
22285        assert!(
22286            body.contains("### Round 1") && !body.contains("first fix"),
22287            "{body}"
22288        );
22289        assert!(
22290            body.contains("### Round 3") && !body.contains("second fix"),
22291            "{body}"
22292        );
22293        assert!(body.contains("### Round 2"), "{body}");
22294    }
22295
22296    const FINAL: &str = "abcdef1234567890";
22297
22298    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22299        let mut r = round_with_notes(round, None);
22300        r.verified_head = verified.map(str::to_owned);
22301        r
22302    }
22303
22304    fn ok_gate(state: &mut RunState, head: Option<&str>) {
22305        state.gate = vec![CommandOutcome {
22306            command: "gate".to_owned(),
22307            code: Some(0),
22308            output_tail: String::new(),
22309            duration_ms: 0,
22310            resource_blocked: false,
22311        }];
22312        state.gate_ran = true;
22313        state.gate_head = head.map(str::to_owned);
22314    }
22315
22316    fn body_at(state: &RunState) -> String {
22317        pr_message_raw(
22318            state,
22319            'A',
22320            None,
22321            crate::pr_outcome::FinalHead::At(FINAL),
22322            ChangeNote::NotAsked,
22323        )
22324        .body
22325    }
22326
22327    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22328
22329    #[test]
22330    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22331        let mut state = review_state(&["feat: x"]);
22332        state.reviews = vec![
22333            round_with_notes(1, Some(SANDBOXED)),
22334            passing_round(2, Some(FINAL)),
22335        ];
22336        ok_gate(&mut state, Some(FINAL));
22337        let body = body_at(&state);
22338        assert!(!body.contains("Handled the PATH case."), "{body}");
22339        assert!(!body.contains("could not commit"), "{body}");
22340        assert!(!body.contains("could not run"), "{body}");
22341        assert!(
22342            body.contains("The fix is committed on the branch"),
22343            "{body}"
22344        );
22345        assert!(!body.contains("sandbox"), "{body}");
22346        assert!(
22347            body.contains("Tests (`verify.e2e`): passed on the final commit"),
22348            "{body}"
22349        );
22350        assert!(
22351            body.contains("Gate (`verify.gate`): passed on the final commit"),
22352            "{body}"
22353        );
22354        // The raw report is untouched in the record.
22355        assert!(
22356            state.reviews[0]
22357                .fix
22358                .as_ref()
22359                .unwrap()
22360                .notes
22361                .contains("could not commit")
22362        );
22363    }
22364
22365    #[test]
22366    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22367        let mut state = review_state(&["feat: x"]);
22368        let mut r1 = round_with_notes(1, Some(SANDBOXED));
22369        r1.fix.as_mut().unwrap().committed = false;
22370        let mut r2 = passing_round(2, Some(FINAL));
22371        r2.e2e[0].code = Some(1);
22372        state.reviews = vec![r1, r2];
22373        let body = body_at(&state);
22374        assert!(!body.contains("could not commit"), "{body}");
22375        assert!(!body.contains("could not run"), "{body}");
22376        assert!(body.contains("This round produced no commit"), "{body}");
22377        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22378        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22379        assert!(!body.contains("passed"), "{body}");
22380    }
22381
22382    #[test]
22383    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22384        let mut state = review_state(&["feat: x"]);
22385        state.reviews = vec![
22386            round_with_notes(1, Some(SANDBOXED)),
22387            passing_round(2, Some("0123456789abcdef")),
22388        ];
22389        ok_gate(&mut state, Some("0123456789abcdef"));
22390        let body = body_at(&state);
22391        assert!(!body.contains("HTTP tests could not run"), "{body}");
22392        // The commit claim is still settled by the structured record.
22393        assert!(!body.contains("could not commit"), "{body}");
22394        assert!(
22395            body.contains("passed on an earlier commit (`0123456`)"),
22396            "{body}"
22397        );
22398        assert!(!body.contains("passed on the final commit"), "{body}");
22399
22400        // A gate recorded before `gate_head` existed is "not recorded".
22401        ok_gate(&mut state, None);
22402        let body = body_at(&state);
22403        assert!(
22404            body.contains("passed on a commit that was not recorded"),
22405            "{body}"
22406        );
22407    }
22408
22409    #[test]
22410    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22411        let state = state_with_summary(
22412            "add retries to the client",
22413            "TITLE: feat: retries\n\nDid it.",
22414        );
22415        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22416        assert_eq!(m.title, "feat: retries");
22417        assert!(
22418            m.body.contains("<summary>Original task</summary>"),
22419            "{}",
22420            m.body
22421        );
22422        assert!(!m.body.contains("Commits under review"), "{}", m.body);
22423        assert!(
22424            m.body
22425                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22426            "{}",
22427            m.body
22428        );
22429    }
22430
22431    #[test]
22432    fn pr_message_review_bounds_a_long_english_subject() {
22433        let long = format!("feat: {}", "word ".repeat(100));
22434        let m = pr_message(&review_state(&[&long]), 'A');
22435        assert!(m.title.starts_with("feat: word"), "{}", m.title);
22436        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22437    }
22438
22439    #[test]
22440    fn pr_message_implementation_run_is_unchanged_by_review_support() {
22441        let state = state_with_summary("add retries\n\ndetails", "- did some things");
22442        let m = pr_message(&state, 'A');
22443        assert_eq!(m.title, "add retries");
22444        assert!(m.body.contains("<summary>Original task</summary>"));
22445        assert!(!m.body.contains("Commits under review"));
22446        assert_eq!(landing_subject_source(&state), state.instruction);
22447    }
22448
22449    #[test]
22450    fn review_run_squash_subject_is_the_change_not_the_prompt() {
22451        let state = review_state(&["feat: the change", "fix: typo"]);
22452        let source = landing_subject_source(&state);
22453        assert_eq!(land::merge_subject("", &source), "feat: the change");
22454        assert_eq!(
22455            land::merge_subject("magi: candidate A (uncommitted work)", &source),
22456            "feat: the change"
22457        );
22458        // An operator's rename still wins.
22459        assert_eq!(
22460            land::merge_subject("feat: renamed by hand", &source),
22461            "feat: renamed by hand"
22462        );
22463        let blank = review_state(&["日本語"]);
22464        assert!(
22465            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22466        );
22467    }
22468
22469    #[test]
22470    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22471        let state = review_state(&["feat: the change"]);
22472        let source = landing_subject_source(&state);
22473        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22474        assert_eq!(
22475            land::merge_subject(landing_title(&state, old), &source),
22476            "feat: the change"
22477        );
22478        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22479        let task = state_with_summary("add retries", "");
22480        assert_eq!(landing_title(&task, old), old);
22481    }
22482
22483    #[test]
22484    fn pr_message_describes_the_change_not_the_task() {
22485        let state = state_with_summary(
22486            "今回やってほしいこと: results projector を直す",
22487            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22488        );
22489        let m = pr_message(&state, 'A');
22490        assert_eq!(m.title, "fix(web): batch the runs list reads");
22491        assert!(
22492            m.body.starts_with("## Summary\n\n- reads run.json once"),
22493            "{}",
22494            m.body
22495        );
22496        assert!(!m.body.contains("TITLE:"), "{}", m.body);
22497        let task_at = m.body.find("今回やってほしいこと").unwrap();
22498        let details_at = m.body.find("<details>").unwrap();
22499        assert!(
22500            details_at < task_at,
22501            "the task lives inside <details>: {}",
22502            m.body
22503        );
22504        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22505        assert!(m.body.contains("magi:candidate-a"));
22506    }
22507
22508    #[test]
22509    fn pr_message_falls_back_to_the_task_without_a_title_line() {
22510        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22511        let m = pr_message(&state, 'A');
22512        assert_eq!(m.title, "add retries");
22513        assert!(
22514            m.body.contains("## Summary\n\n- did some things"),
22515            "{}",
22516            m.body
22517        );
22518
22519        let none = RunState::new(
22520            PathBuf::from("/repo"),
22521            "main".to_owned(),
22522            "abc1234".to_owned(),
22523            "add retries".to_owned(),
22524            Config::default(),
22525        );
22526        let m = pr_message(&none, 'A');
22527        assert_eq!(m.title, "add retries");
22528        assert!(!m.body.contains("## Summary"), "{}", m.body);
22529    }
22530
22531    #[test]
22532    fn pr_message_refuses_the_candidate_commit_subject() {
22533        for bad in [
22534            "TITLE: magi: candidate A (uncommitted work)",
22535            "TITLE: chore: stuff (uncommitted work)",
22536            "TITLE:   ",
22537        ] {
22538            let state = state_with_summary("add retries", bad);
22539            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22540        }
22541    }
22542
22543    #[test]
22544    fn pr_message_bounds_a_very_long_task_and_title() {
22545        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22546        let state = state_with_summary(&long, "- nothing");
22547        let m = pr_message(&state, 'A');
22548        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22549        assert!(!m.title.contains('\n'));
22550
22551        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22552        let m = pr_message(&state, 'A');
22553        assert!(m.title.starts_with("feat: "));
22554        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22555        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22556    }
22557
22558    fn long_title_of(instruction: &str) -> String {
22559        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22560    }
22561
22562    #[test]
22563    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22564        let first = "Make the landing path keep a readable title for long tasks";
22565        let line = format!(
22566            "{first}. {}",
22567            "And then keep going with more words ".repeat(20)
22568        );
22569        let t = long_title_of(&line);
22570        assert_eq!(t, first);
22571        assert!(!t.starts_with("chore: land"));
22572    }
22573
22574    #[test]
22575    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22576        let line = "word ".repeat(200);
22577        let t = long_title_of(&line);
22578        assert!(t.ends_with("word..."), "{t}");
22579        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22580    }
22581
22582    #[test]
22583    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22584        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22585            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22586        }
22587    }
22588
22589    #[test]
22590    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22591        let mut state = state_with_summary("再試行を追加する", "- nothing");
22592        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22593        state.config.graph.github_language = "ja".into();
22594        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22595    }
22596
22597    #[test]
22598    fn pr_message_title_limit_is_exact() {
22599        let at = "a".repeat(PR_TITLE_MAX);
22600        assert_eq!(long_title_of(&at), at);
22601        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22602        assert!(over.ends_with("..."), "{over}");
22603        assert_eq!(over.chars().count(), PR_TITLE_MAX);
22604    }
22605
22606    #[test]
22607    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22608        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22609        let t = long_title_of(&line);
22610        assert!(t.ends_with("..."), "{t}");
22611        assert!(t.is_ascii(), "{t}");
22612    }
22613
22614    #[test]
22615    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22616        let line = format!(
22617            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22618            "plus more filler words ".repeat(20)
22619        );
22620        let t = long_title_of(&line);
22621        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22622    }
22623
22624    #[test]
22625    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22626        let line = format!("\n\n# {}", "title words ".repeat(40));
22627        let state = state_with_summary(&line, "- nothing");
22628        let m = pr_message(&state, 'A');
22629        assert!(m.title.starts_with("title words"), "{}", m.title);
22630        assert_eq!(
22631            land::merge_subject(&m.title, &landing_subject_source(&state)),
22632            m.title
22633        );
22634        // An operator's rename wins untouched.
22635        assert_eq!(
22636            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22637            "feat: renamed by hand"
22638        );
22639    }
22640
22641    #[tokio::test]
22642    async fn github_text_rewrite_is_bounded_and_falls_back() {
22643        crate::run::pin_test_home();
22644        for (reply, accepted) in [
22645            (
22646                "TITLE: fix: retries\nAdd retries for failed requests.",
22647                true,
22648            ),
22649            (
22650                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22651                false,
22652            ),
22653            (
22654                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22655                true,
22656            ),
22657            (
22658                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22659                true,
22660            ),
22661            (
22662                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22663                true,
22664            ),
22665            (
22666                "TITLE: fix: retries\n日本語の説明です。 token=secret",
22667                false,
22668            ),
22669            ("TITLE: fix: retries\n", false),
22670        ] {
22671            let dir = tempfile::tempdir().unwrap();
22672            let mut runner = runner_at(RunStatus::Gating);
22673            runner.state = state_with_summary(
22674                "add retries",
22675                "TITLE: fix: retries\n日本語の説明を書きます。",
22676            );
22677            runner.state.repo = dir.path().to_owned();
22678            runner.state.candidates[0].worktree = dir.path().to_owned();
22679            let mut author = spec("alpha");
22680            author.command = vec![
22681                "sh".into(),
22682                "-c".into(),
22683                "printf '%s' \"$REWRITE_REPLY\"".into(),
22684            ];
22685            author.env.insert("REWRITE_REPLY".into(), reply.into());
22686            runner.state.config.agents = vec![author];
22687            seed_public_summary(&mut runner);
22688            let winner = runner.state.candidates[0].clone();
22689            let first = runner
22690                .guarded_pr_message(&winner, None, true)
22691                .await
22692                .unwrap();
22693            let message = if accepted {
22694                first.expect("a passing rewrite never asks")
22695            } else {
22696                // Withheld: the run parks on a question and posts nothing yet.
22697                assert!(first.is_none());
22698                assert!(runner.state.parked);
22699                let store = ask::Questions::open();
22700                let asked: Vec<_> = store
22701                    .list()
22702                    .into_iter()
22703                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22704                    .collect();
22705                assert_eq!(asked.len(), 1);
22706                assert!(!asked[0].detail.contains("secret"));
22707                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22708                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22709                assert_eq!(
22710                    asked[0]
22711                        .choices
22712                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
22713                    rewrite.is_some()
22714                );
22715                // Silence: the question lapses and the neutral text is used,
22716                // without a second rewrite call or a second question.
22717                store
22718                    .update(&asked[0].id, |q| {
22719                        q.abandon("test");
22720                        Ok(())
22721                    })
22722                    .unwrap();
22723                let again = runner
22724                    .guarded_pr_message(&winner, None, true)
22725                    .await
22726                    .unwrap()
22727                    .expect("an abandoned question falls back");
22728                let n = store
22729                    .list()
22730                    .into_iter()
22731                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22732                    .count();
22733                assert_eq!(n, 1, "asked once per rejected text");
22734                again
22735            };
22736            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22737            assert!(!message.body.contains("secret"));
22738            assert!(!message.body.contains("prefix"));
22739            assert!(!message.title.contains("secret"));
22740            assert_eq!(
22741                message.body.contains("Add retries for failed requests."),
22742                accepted
22743            );
22744            assert_eq!(
22745                runner.state.seats["github-text-rewrite"].turns, 1,
22746                "only one rewrite invocation"
22747            );
22748            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22749            if accepted {
22750                assert!(runner.state.github_text.is_none());
22751                assert!(!runner.state.parked);
22752                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22753                assert!(message.body.contains("## Final verification"));
22754                assert!(message.body.contains("Evidence: final diff"));
22755                assert!(
22756                    message
22757                        .body
22758                        .contains(&format!("magi:run/{}", runner.state.id))
22759                );
22760            }
22761            if !accepted {
22762                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22763                assert!(
22764                    message
22765                        .body
22766                        .contains(&format!("magi:run/{}", runner.state.id))
22767                );
22768            }
22769        }
22770    }
22771
22772    fn scripted(id: &str, reply: &str) -> AgentSpec {
22773        let mut a = spec(id);
22774        a.command = vec![
22775            "sh".into(),
22776            "-c".into(),
22777            "printf '%s' \"$REWRITE_REPLY\"".into(),
22778        ];
22779        a.env.insert("REWRITE_REPLY".into(), reply.into());
22780        a
22781    }
22782
22783    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22784    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22785
22786    /// Gate tests inject generated public output, not internal reports.
22787    fn seed_public_summary(runner: &mut Runner) {
22788        let repo = &runner.state.repo;
22789        let g = |args: &[&str]| {
22790            let out = std::process::Command::new("git")
22791                .args(args)
22792                .current_dir(repo)
22793                .quiet()
22794                .output()
22795                .unwrap();
22796            assert!(out.status.success(), "git {args:?}");
22797            String::from_utf8_lossy(&out.stdout).trim().to_owned()
22798        };
22799        g(&["init", "-q", "-b", "main"]);
22800        g(&[
22801            "-c",
22802            "user.name=Test",
22803            "-c",
22804            "user.email=test@example.invalid",
22805            "commit",
22806            "-q",
22807            "--allow-empty",
22808            "-m",
22809            "feat: fixture",
22810        ]);
22811        let head = g(&["rev-parse", "HEAD"]);
22812        g(&["update-ref", "refs/remotes/origin/main", &head]);
22813        let evidence = crate::pr_outcome::summary_evidence(
22814            &runner.state,
22815            crate::pr_outcome::FinalHead::At(&head),
22816        );
22817        let generated = &runner.state.candidates[0].summary;
22818        runner.state.change_summary = Some(crate::run::ChangeSummary {
22819            base: head.clone(),
22820            head,
22821            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22822            title: summary_title(generated).unwrap(),
22823            body: summary_without_title(generated),
22824        });
22825    }
22826
22827    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22828        let mut runner = runner_at(RunStatus::Gating);
22829        runner.state = state_with_summary("add retries", JA_SUMMARY);
22830        if review_only {
22831            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22832        }
22833        runner.state.repo = dir.to_owned();
22834        runner.state.candidates[0].worktree = dir.to_owned();
22835        runner.state.config.agents = agents;
22836        seed_public_summary(&mut runner);
22837        runner
22838    }
22839
22840    fn gate_questions(runner: &Runner) -> usize {
22841        ask::Questions::open()
22842            .list()
22843            .into_iter()
22844            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22845            .count()
22846    }
22847
22848    #[tokio::test]
22849    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22850        crate::run::pin_test_home();
22851        let dir = tempfile::tempdir().unwrap();
22852        let mut runner = rewrite_runner(
22853            dir.path(),
22854            true,
22855            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22856        );
22857        runner.state.config.roles.language_judge =
22858            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22859        let winner = runner.state.candidates[0].clone();
22860        let msg = runner
22861            .guarded_pr_message(&winner, None, true)
22862            .await
22863            .unwrap()
22864            .expect("an English rewrite posts without asking");
22865        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22866        assert!(msg.body.contains("<details>kept</details>"));
22867        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22868        assert_eq!(gate_questions(&runner), 0);
22869        assert!(runner.state.github_text.is_none());
22870        assert!(!runner.state.seats.contains_key("impl-A"));
22871        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22872    }
22873
22874    #[tokio::test]
22875    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22876        crate::run::pin_test_home();
22877        let dir = tempfile::tempdir().unwrap();
22878        let mut runner = rewrite_runner(
22879            dir.path(),
22880            false,
22881            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22882        );
22883        runner.state.config.roles.language_judge =
22884            Some(crate::config::AgentChoice::One("beta".into()));
22885        runner.state.seats.insert(
22886            "impl-A".into(),
22887            SeatState::new("impl-A", "alpha", runner.state.seed),
22888        );
22889        let winner = runner.state.candidates[0].clone();
22890        let msg = runner
22891            .guarded_pr_message(&winner, None, true)
22892            .await
22893            .unwrap()
22894            .expect("rewritten");
22895        assert!(msg.body.contains("<details>kept</details>"));
22896        assert_eq!(runner.state.seats["impl-A"].turns, 0);
22897        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22898        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22899    }
22900
22901    #[tokio::test]
22902    async fn rewrite_reuses_a_resumable_implementer_session() {
22903        crate::run::pin_test_home();
22904        let dir = tempfile::tempdir().unwrap();
22905        let mut runner = rewrite_runner(
22906            dir.path(),
22907            false,
22908            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22909        );
22910        runner.state.config.roles.language_judge =
22911            Some(crate::config::AgentChoice::One("beta".into()));
22912        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22913        seat.turns = 1;
22914        runner.state.seats.insert("impl-A".into(), seat);
22915        let winner = runner.state.candidates[0].clone();
22916        runner
22917            .guarded_pr_message(&winner, None, true)
22918            .await
22919            .unwrap()
22920            .expect("rewritten");
22921        assert_eq!(runner.state.seats["impl-A"].turns, 2);
22922        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22923    }
22924
22925    #[tokio::test]
22926    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22927        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
22928        // session, was out of quota, and nothing else was asked.
22929        crate::run::pin_test_home();
22930        let dir = tempfile::tempdir().unwrap();
22931        let mut exhausted = spec("alpha");
22932        exhausted.command = vec!["false".into()];
22933        let mut runner = rewrite_runner(
22934            dir.path(),
22935            false,
22936            vec![exhausted, scripted("beta", EN_REWRITE)],
22937        );
22938        // The implementer's own agent is in the role too: it is not asked twice.
22939        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22940            "alpha".into(),
22941            "beta".into(),
22942        ]));
22943        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22944        seat.turns = 1;
22945        runner.state.seats.insert("impl-A".into(), seat);
22946        let winner = runner.state.candidates[0].clone();
22947        let msg = runner
22948            .guarded_pr_message(&winner, None, true)
22949            .await
22950            .unwrap()
22951            .expect("the role rewrites after the implementer fails");
22952        assert!(msg.body.contains("`src/net.rs`"));
22953        assert_eq!(
22954            runner.state.seats["impl-A"].turns, 2,
22955            "the implementer was tried once"
22956        );
22957        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22958        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22959    }
22960
22961    #[tokio::test]
22962    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22963        crate::run::pin_test_home();
22964        let dir = tempfile::tempdir().unwrap();
22965        let mut broken = spec("bad");
22966        broken.command = vec!["false".into()];
22967        let mut runner = rewrite_runner(
22968            dir.path(),
22969            true,
22970            vec![
22971                broken,
22972                scripted("noformat", "Add retries."),
22973                scripted("good", EN_REWRITE),
22974            ],
22975        );
22976        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22977            "bad".into(),
22978            "noformat".into(),
22979            "good".into(),
22980        ]));
22981        let winner = runner.state.candidates[0].clone();
22982        let msg = runner
22983            .guarded_pr_message(&winner, None, true)
22984            .await
22985            .unwrap()
22986            .expect("the third agent rewrites");
22987        assert!(msg.body.contains("`src/net.rs`"));
22988        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
22989        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
22990    }
22991
22992    #[tokio::test]
22993    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
22994        crate::run::pin_test_home();
22995        // (agents, role): still Japanese; no TITLE; nothing resolvable.
22996        for (agents, role, saved) in [
22997            (
22998                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
22999                Some("beta"),
23000                true,
23001            ),
23002            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23003            (Vec::new(), None, false),
23004        ] {
23005            let dir = tempfile::tempdir().unwrap();
23006            let mut runner = rewrite_runner(dir.path(), true, agents);
23007            runner.state.config.roles.language_judge =
23008                role.map(|r| crate::config::AgentChoice::One(r.into()));
23009            let winner = runner.state.candidates[0].clone();
23010            let first = runner
23011                .guarded_pr_message(&winner, None, true)
23012                .await
23013                .unwrap();
23014            assert!(first.is_none(), "withheld text parks the run");
23015            assert!(runner.state.parked);
23016            assert_eq!(gate_questions(&runner), 1);
23017            let g = runner.state.github_text.as_ref().unwrap();
23018            assert_eq!(g.rewrite.is_some(), saved);
23019            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23020        }
23021    }
23022
23023    #[tokio::test]
23024    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23025        crate::run::pin_test_home();
23026        let dir = tempfile::tempdir().unwrap();
23027        let mut runner = runner_at(RunStatus::Gating);
23028        runner.state =
23029            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23030        runner.state.repo = dir.path().to_owned();
23031        runner.state.candidates[0].worktree = dir.path().to_owned();
23032        let mut author = spec("alpha");
23033        author.command = vec![
23034            "sh".into(),
23035            "-c".into(),
23036            "printf '%s' \"$REWRITE_REPLY\"".into(),
23037        ];
23038        author.env.insert(
23039            "REWRITE_REPLY".into(),
23040            format!(
23041                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23042                "English sentence. ".repeat(600)
23043            ),
23044        );
23045        runner.state.config.agents = vec![author];
23046        seed_public_summary(&mut runner);
23047        let winner = runner.state.candidates[0].clone();
23048        assert!(
23049            runner
23050                .guarded_pr_message(&winner, None, true)
23051                .await
23052                .unwrap()
23053                .is_none()
23054        );
23055        let record = runner.state.github_text.clone().unwrap();
23056        let rewrite = record.rewrite.clone().unwrap();
23057        assert_eq!(rewrite.failed, ["body-language"]);
23058        assert!(
23059            rewrite
23060                .body
23061                .ends_with(&format!("magi:run/{}", runner.state.id))
23062        );
23063        let store = ask::Questions::open();
23064        let q = store.get(record.question.as_ref().unwrap()).unwrap();
23065        assert!(
23066            q.choices
23067                .contains(&crate::github_text::USE_REWRITE.to_owned())
23068        );
23069        for label in [
23070            "ORIGINAL",
23071            "FALLBACK",
23072            "REWRITE",
23073            "Rules still failed: body-language",
23074            "truncated",
23075        ] {
23076            assert!(q.detail.contains(label), "{label}");
23077        }
23078        let artifact = std::fs::read_to_string(crate::run::artifact_path(
23079            &runner.state,
23080            &crate::github_text::artifact_name(&record.fingerprint),
23081        ))
23082        .unwrap();
23083        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23084        // Restart from serialized state, then retry after a bad owner title.
23085        runner.state =
23086            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23087        store
23088            .update(&q.id, |q| {
23089                q.say("まだ日本語")?;
23090                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23091            })
23092            .unwrap();
23093        assert!(
23094            runner
23095                .guarded_pr_message(&winner, None, true)
23096                .await
23097                .unwrap()
23098                .is_none()
23099        );
23100        let retry = runner.state.github_text.as_ref().unwrap();
23101        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
23102        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
23103        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
23104        assert!(
23105            q.posting_comparison
23106                .as_ref()
23107                .unwrap()
23108                .matches_detail(&q.detail)
23109        );
23110        store
23111            .update(&q.id, |q| {
23112                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23113            })
23114            .unwrap();
23115        let message = runner
23116            .guarded_pr_message(&winner, None, true)
23117            .await
23118            .unwrap()
23119            .unwrap();
23120        assert_eq!(message.title, rewrite.title);
23121        assert_eq!(message.body, rewrite.body);
23122        assert_eq!(
23123            runner.state.github_text.as_ref().unwrap().choice,
23124            Some(GithubTextChoice::Rewrite)
23125        );
23126        let again = runner
23127            .guarded_pr_message(&winner, None, true)
23128            .await
23129            .unwrap()
23130            .unwrap();
23131        assert_eq!(again.title, message.title);
23132        assert_eq!(again.body, message.body);
23133        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23134        assert!(
23135            !runner
23136                .state
23137                .events
23138                .iter()
23139                .any(|e| e.message.contains(&rewrite.title)
23140                    || e.message.contains("English sentence."))
23141        );
23142    }
23143
23144    #[tokio::test]
23145    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
23146        crate::run::pin_test_home();
23147        let dir = tempfile::tempdir().unwrap();
23148        let mut runner = runner_at(RunStatus::Gating);
23149        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23150        runner.state.repo = dir.path().to_owned();
23151        runner.state.candidates[0].worktree = dir.path().to_owned();
23152        seed_public_summary(&mut runner);
23153        let winner = runner.state.candidates[0].clone();
23154        assert!(
23155            runner
23156                .guarded_pr_message(&winner, None, true)
23157                .await
23158                .unwrap()
23159                .is_none()
23160        );
23161        let record = runner.state.github_text.as_ref().unwrap();
23162        assert!(record.rewrite.is_none());
23163        // Records written before the additive field deserialize with no rewrite.
23164        let mut old = serde_json::to_value(record).unwrap();
23165        old.as_object_mut().unwrap().remove("rewrite");
23166        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23167        assert!(old.rewrite.is_none());
23168        let id = record.question.clone().unwrap();
23169        ask::Questions::open()
23170            .update(&id, |q| {
23171                q.choices.push(crate::github_text::USE_REWRITE.into());
23172                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23173            })
23174            .unwrap();
23175        let message = runner
23176            .guarded_pr_message(&winner, None, true)
23177            .await
23178            .unwrap()
23179            .unwrap();
23180        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23181        assert_eq!(
23182            runner.state.github_text.as_ref().unwrap().choice,
23183            Some(GithubTextChoice::Fallback)
23184        );
23185    }
23186
23187    #[tokio::test]
23188    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23189        crate::run::pin_test_home();
23190        let dir = tempfile::tempdir().unwrap();
23191        let mut runner = runner_at(RunStatus::Gating);
23192        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23193        runner.state.repo = dir.path().to_owned();
23194        runner.state.candidates[0].worktree = dir.path().to_owned();
23195        seed_public_summary(&mut runner);
23196        let winner = runner.state.candidates[0].clone();
23197        assert!(
23198            runner
23199                .guarded_pr_message(&winner, None, true)
23200                .await
23201                .unwrap()
23202                .is_none()
23203        );
23204        let store = ask::Questions::open();
23205        let find = |run: &str| {
23206            store
23207                .list()
23208                .into_iter()
23209                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23210                .unwrap()
23211        };
23212        let run = runner.state.id.clone();
23213        let q = find(&run);
23214        let fp = runner
23215            .state
23216            .github_text
23217            .as_ref()
23218            .unwrap()
23219            .fingerprint
23220            .clone();
23221        let name = crate::github_text::artifact_name(&fp);
23222        let path = crate::run::artifact_path(&runner.state, &name);
23223        assert!(q.detail.contains(&path.display().to_string()));
23224        assert!(q.detail.contains("title-language: title"));
23225        let saved = std::fs::read_to_string(&path).expect("artifact written");
23226        assert!(saved.contains("日本語のタイトル"));
23227        assert!(q.cwd.is_none());
23228        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23229        // A bad replacement is refused and asked about once more.
23230        let say = |id: &str, text: &str, choice: &str| {
23231            store
23232                .update(id, |q| {
23233                    q.thread.push(ask::Turn {
23234                        who: ask::Who::Operator,
23235                        body: text.to_owned(),
23236                        at: jiff::Timestamp::now(),
23237                        note: None,
23238                    });
23239                    q.answer(ask::Answer::Choice(choice.to_owned()))
23240                })
23241                .unwrap();
23242        };
23243        say(&q.id, "まだ日本語", "use my text");
23244        assert!(
23245            runner
23246                .guarded_pr_message(&winner, None, true)
23247                .await
23248                .unwrap()
23249                .is_none()
23250        );
23251        let second = find(&run);
23252        assert_ne!(second.id, q.id);
23253        assert!(second.detail.contains("did not pass"));
23254        // The retry still compares the original text; the rejected
23255        // replacement is named by category only.
23256        assert!(second.detail.contains("日本語のタイトル"));
23257        assert!(!second.detail.contains("まだ日本語"));
23258        assert!(second.detail.contains(&path.display().to_string()));
23259        say(&second.id, "fix: retry failed requests", "use my text");
23260        let message = runner
23261            .guarded_pr_message(&winner, None, true)
23262            .await
23263            .unwrap()
23264            .expect("a vetted replacement is posted");
23265        assert_eq!(message.title, "fix: retry failed requests");
23266        assert_eq!(
23267            runner
23268                .state
23269                .github_text
23270                .as_ref()
23271                .unwrap()
23272                .chosen_title
23273                .as_deref(),
23274            Some("fix: retry failed requests")
23275        );
23276        assert!(runner.state.github_text.as_ref().unwrap().resolved);
23277        // Resumed again: the saved decision is applied, nothing is re-asked.
23278        let again = runner
23279            .guarded_pr_message(&winner, None, true)
23280            .await
23281            .unwrap()
23282            .unwrap();
23283        assert_eq!(again.title, message.title);
23284        assert_eq!(
23285            store
23286                .list()
23287                .into_iter()
23288                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23289                .count(),
23290            2
23291        );
23292    }
23293
23294    #[test]
23295    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23296        // What magi itself writes stays English under any configured language,
23297        // so a future localisation of these headings fails here. (The agents'
23298        // own text is also checked by the posting gate.)
23299        let mut state = state_with_summary(
23300            "add retries",
23301            "TITLE: fix(web): batch reads\n- reads run.json once",
23302        );
23303        state.config.graph.language = "ja".to_owned();
23304        let m = pr_message(&state, 'A');
23305        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23306        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23307
23308        // The task is the operator's own text: it goes in untouched, and the
23309        // fallback title (no summary) may be in its language too.
23310        let task = "今回やってほしいこと: results projector を直す";
23311        let mut state = state_with_summary(task, "- no title line");
23312        state.config.graph.language = "ja".to_owned();
23313        let m = pr_message(&state, 'A');
23314        assert_eq!(
23315            m.title,
23316            format!("chore: land candidate A of run {}", state.id)
23317        );
23318        assert!(
23319            m.body.contains(&format!(
23320                "<summary>Original task</summary>\n\n{task}\n\n</details>"
23321            )),
23322            "{}",
23323            m.body
23324        );
23325    }
23326
23327    #[test]
23328    fn pr_message_scrubs_home_paths_and_addresses() {
23329        let state = state_with_summary(
23330            "fix it in /Users/someone/src/x",
23331            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23332        );
23333        let m = pr_message(&state, 'A');
23334        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23335            assert!(!m.body.contains(leak), "{}", m.body);
23336        }
23337        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23338    }
23339
23340    #[test]
23341    fn pr_message_survives_a_task_that_closes_details() {
23342        let state = state_with_summary("a </details> b", "TITLE: fix: x");
23343        let m = pr_message(&state, 'A');
23344        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23345    }
23346
23347    #[test]
23348    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23349        let cmd = manual_merge_command(
23350            MergeStyle::Squash,
23351            Path::new("/repo"),
23352            "b",
23353            "fix: \"quoted\" $(x) `y`\n\nbody",
23354        );
23355        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23356    }
23357
23358    #[test]
23359    fn manual_merge_command_matches_the_configured_style() {
23360        let repo = Path::new("/repo");
23361        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23362
23363        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23364        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23365
23366        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23367        assert_eq!(
23368            squash,
23369            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23370             \"Merge magi run 0832 (candidate A)\""
23371        );
23372
23373        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23374        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23375    }
23376
23377    #[test]
23378    fn a_nudge_gets_a_quarter_of_the_budget() {
23379        // The judge and implement budgets magi ships with.
23380        assert_eq!(retry_budget(secs(1200), true), secs(300));
23381        assert_eq!(retry_budget(secs(3600), true), secs(900));
23382    }
23383
23384    #[test]
23385    fn a_resent_prompt_keeps_the_whole_budget() {
23386        // The seat kept no context, so the retry is the original job again and
23387        // shortening it would only guarantee a second failure.
23388        assert_eq!(retry_budget(secs(1200), false), secs(1200));
23389        assert_eq!(retry_budget(secs(60), false), secs(60));
23390    }
23391
23392    #[test]
23393    fn the_floor_never_exceeds_the_original_budget() {
23394        // A short configured timeout must not be *raised* by the floor: the
23395        // operator asked for a bound, and a retry may not outlast the attempt
23396        // it is retrying.
23397        assert_eq!(retry_budget(secs(60), true), secs(60));
23398        assert_eq!(retry_budget(secs(480), true), secs(120));
23399        assert_eq!(retry_budget(secs(0), true), secs(0));
23400    }
23401
23402    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23403        agent::CommandEvidence {
23404            id: "item1".to_owned(),
23405            description: "cargo test".to_owned(),
23406            exit_code,
23407            result_summary: String::new(),
23408            source: "codex".to_owned(),
23409        }
23410    }
23411
23412    #[test]
23413    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23414        // No evidence is not the same fact as unconfirmed evidence: a
23415        // backend with no adapter, or a reply that ran no commands at all,
23416        // must not be misread as carrying a dangling job.
23417        assert!(!has_unconfirmed_command(&[]));
23418    }
23419
23420    #[test]
23421    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23422        // Deliberately not a check on the exit code's *value*: a fixer
23423        // legitimately runs something that fails mid-iteration before it
23424        // succeeds, and that must never by itself reopen a valid report.
23425        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23426        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23427        assert!(!has_unconfirmed_command(&[
23428            evidence(Some(0)),
23429            evidence(Some(101))
23430        ]));
23431    }
23432
23433    #[test]
23434    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23435        assert!(has_unconfirmed_command(&[
23436            evidence(Some(0)),
23437            evidence(None)
23438        ]));
23439    }
23440
23441    #[test]
23442    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23443        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23444        assert_eq!(
23445            verified_noop_claim(true, &[], text).as_deref(),
23446            Some("already fixed by b32cfc4, on main.")
23447        );
23448    }
23449
23450    #[test]
23451    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23452        // A timeout or a bad exit code reads as the ordinary loss it is,
23453        // whatever the reply's own prose claims.
23454        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23455        assert!(verified_noop_claim(false, &[], text).is_none());
23456    }
23457
23458    #[test]
23459    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23460        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23461        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23462        // A confirmed command alongside the marker is fine.
23463        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23464    }
23465
23466    #[test]
23467    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23468        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23469    }
23470
23471    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
23472    /// pair, in order, labelled A, B, C, ...
23473    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23474        runner.state.candidates = shape
23475            .iter()
23476            .enumerate()
23477            .map(|(i, &(empty, verified))| Candidate {
23478                index: i,
23479                label: (b'A' + i as u8) as char,
23480                agent: "sonnet".to_owned(),
23481                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23482                worktree: PathBuf::from(format!("/wt/{i}")),
23483                summary: String::new(),
23484                stat: String::new(),
23485                files: 0,
23486                commits: 0,
23487                empty,
23488                failed: None,
23489                verified_noop: verified.map(str::to_owned),
23490                duration_ms: 0,
23491                folded: false,
23492            })
23493            .collect();
23494    }
23495
23496    #[test]
23497    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23498        ask_test_home();
23499        let mut runner = runner_at(RunStatus::Implementing);
23500        set_candidates(
23501            &mut runner,
23502            &[
23503                (true, Some("already on main at b32cfc4")),
23504                (true, Some("same fix, see the existing test")),
23505            ],
23506        );
23507
23508        runner
23509            .after_implement()
23510            .expect("a verified no-op is not an error");
23511
23512        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23513    }
23514
23515    #[test]
23516    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23517        ask_test_home();
23518        let mut runner = runner_at(RunStatus::Implementing);
23519        // Candidate A declares a verified no-op; candidate B simply wrote
23520        // nothing and said nothing about why. One candidate's claim is not
23521        // the whole run's agreement.
23522        set_candidates(
23523            &mut runner,
23524            &[(true, Some("already on main at b32cfc4")), (true, None)],
23525        );
23526
23527        let err = runner
23528            .after_implement()
23529            .expect_err("an unverified empty candidate must still fail the run");
23530
23531        assert!(
23532            err.to_string().contains("no candidate produced a change"),
23533            "{err}"
23534        );
23535        assert_eq!(runner.state.status, RunStatus::Failed);
23536    }
23537
23538    #[test]
23539    fn after_implement_still_fails_an_ordinary_all_empty_run() {
23540        ask_test_home();
23541        let mut runner = runner_at(RunStatus::Implementing);
23542        set_candidates(&mut runner, &[(true, None), (true, None)]);
23543
23544        let err = runner
23545            .after_implement()
23546            .expect_err("no candidate declared anything; this is an ordinary failure");
23547
23548        assert!(
23549            err.to_string().contains("no candidate produced a change"),
23550            "{err}"
23551        );
23552        assert_eq!(runner.state.status, RunStatus::Failed);
23553    }
23554}
23555
23556/// A graft rollback whose restore could not be confirmed (see
23557/// [`Runner::graft`]).
23558#[derive(Debug)]
23559struct GraftRestoreFailed(String);
23560
23561impl std::fmt::Display for GraftRestoreFailed {
23562    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23563        write!(
23564            f,
23565            "could not restore the winner to its pre-graft commit: {}",
23566            self.0
23567        )
23568    }
23569}
23570
23571impl std::error::Error for GraftRestoreFailed {}
23572
23573/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
23574/// not the agent: they never draw on an agent's own timeout, and running out of
23575/// them never reads as an agent failure.
23576#[cfg(test)]
23577mod resource_wait_tests {
23578    use super::*;
23579    use std::collections::BTreeMap;
23580
23581    fn sh(id: &str, script: &str) -> AgentSpec {
23582        AgentSpec {
23583            id: id.to_owned(),
23584            kind: crate::config::AgentKind::Command,
23585            model: None,
23586            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23587            extra_args: Vec::new(),
23588            env: BTreeMap::new(),
23589            prompt_delivery: None,
23590            max_turn_secs: None,
23591        }
23592    }
23593
23594    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23595        SeatJob {
23596            seat: SeatState::new("impl-A", &spec.id, 1),
23597            spec,
23598            cwd: dir.to_path_buf(),
23599            prompt: "do it".to_owned(),
23600            timeout,
23601            allow_write: true,
23602            sessions: false,
23603            artifacts: dir.join("art"),
23604            stem: "impl-A".to_owned(),
23605            handover: None,
23606        }
23607    }
23608
23609    fn state(resource_wait_secs: u64) -> RunState {
23610        let mut config = Config::default();
23611        config.graph.timeout_resource_wait = resource_wait_secs;
23612        RunState::new(
23613            PathBuf::from("/repo"),
23614            "main".to_owned(),
23615            "abc1234".to_owned(),
23616            "task".to_owned(),
23617            config,
23618        )
23619    }
23620
23621    fn hold(cache: &Path) -> crate::cache::Guard {
23622        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23623        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23624            crate::cache::AcquireOutcome::Acquired(g) => g,
23625            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23626        }
23627    }
23628
23629    async fn run(
23630        jobs: Vec<SeatJob>,
23631        sem: Arc<Semaphore>,
23632        cache: Option<&Path>,
23633        st: &mut RunState,
23634    ) -> Vec<(usize, SeatState, AgentOutcome)> {
23635        let prompts = Prompts::default();
23636        let ctx = WaveCtx {
23637            run: "r",
23638            node: "implement",
23639            prompts: &prompts,
23640            cache,
23641            round: None,
23642            carry_seats: false,
23643            pauses: None,
23644        };
23645        wave(jobs, sem, &ctx, st, 0).await
23646    }
23647
23648    #[tokio::test]
23649    async fn verification_cache_wait_preserves_the_command_budget() {
23650        crate::run::pin_test_home();
23651        let tmp = tempfile::tempdir().unwrap();
23652        let cache = tmp.path().join("target");
23653        let guard = hold(&cache);
23654        let release = tokio::spawn(async move {
23655            tokio::time::sleep(Duration::from_millis(100)).await;
23656            guard.release();
23657        });
23658        let mut st = state(10);
23659        let budget = Duration::from_millis(50);
23660        let wait = repository_wait(&st, None);
23661        let (outcomes, _) = with_cache_lease(
23662            &mut st,
23663            Some(&cache),
23664            "gate",
23665            "gate",
23666            tmp.path(),
23667            "abc1234",
23668            budget,
23669            "test gate",
23670            &wait,
23671            |_, available| async move {
23672                assert_eq!(
23673                    available, budget,
23674                    "contention must not shorten verification"
23675                );
23676                (Vec::new(), false, Vec::new())
23677            },
23678        )
23679        .await
23680        .unwrap();
23681        release.await.unwrap();
23682        assert!(outcomes.is_empty(), "{outcomes:?}");
23683        assert!(
23684            st.events
23685                .iter()
23686                .any(|event| event.message.contains("waiting for"))
23687        );
23688        assert!(st.resource_wait.is_none());
23689        // The verification body no longer owns the lease after returning.
23690        hold(&cache).release();
23691    }
23692
23693    #[tokio::test]
23694    async fn verification_cache_wait_expires_without_starting_commands() {
23695        crate::run::pin_test_home();
23696        let tmp = tempfile::tempdir().unwrap();
23697        let cache = tmp.path().join("target");
23698        let _held = hold(&cache);
23699        let mut st = state(0);
23700        let wait = repository_wait(&st, None);
23701        let (outcomes, _) = with_cache_lease(
23702            &mut st,
23703            Some(&cache),
23704            "gate",
23705            "gate",
23706            tmp.path(),
23707            "abc1234",
23708            Duration::from_secs(60),
23709            "test gate",
23710            &wait,
23711            |_, _| async { panic!("commands must not start without the cache lease") },
23712        )
23713        .await
23714        .unwrap();
23715        assert_eq!(outcomes.len(), 1);
23716        assert!(outcomes[0].resource_blocked);
23717        assert!(st.resource_wait.is_none());
23718        assert!(matches!(
23719            st.resource_block.unwrap().cause,
23720            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23721        ));
23722    }
23723
23724    #[tokio::test]
23725    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23726        crate::run::pin_test_home();
23727        let tmp = tempfile::tempdir().unwrap();
23728        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23729        let cache = tmp.path().join("cache");
23730        let mut state = state(21600);
23731        let wait = repository_wait(&state, None);
23732        let (outcomes, _) = with_cache_lease(
23733            &mut state,
23734            Some(&cache),
23735            "gate",
23736            "gate",
23737            tmp.path(),
23738            "head",
23739            Duration::from_secs(60),
23740            "gate",
23741            &wait,
23742            |_, _| async { panic!("freshness failure must not start verification") },
23743        )
23744        .await
23745        .unwrap();
23746        assert_eq!(outcomes.len(), 1);
23747        assert!(outcomes[0].resource_blocked);
23748        let block = state.resource_block.unwrap();
23749        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23750        assert!(!block.description().contains("timed out"));
23751    }
23752
23753    #[tokio::test]
23754    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23755        crate::run::pin_test_home();
23756        let tmp = tempfile::tempdir().unwrap();
23757        let cache = tmp.path().join("cache");
23758        let _held = hold(&cache);
23759        let mut state = state(21600);
23760        let pause = Pause::new();
23761        let interrupt = Pause::new();
23762        let cause = ParkCause::Operator;
23763        pause.park_for(cause.clone());
23764        let prompts = Prompts::default();
23765        let ctx = WaveCtx {
23766            run: "test",
23767            node: "fix",
23768            prompts: &prompts,
23769            cache: Some(&cache),
23770            round: None,
23771            carry_seats: false,
23772            pauses: Some((&pause, &interrupt)),
23773        };
23774        let result = wave(
23775            vec![job(
23776                sh("writer", "exit 99"),
23777                tmp.path(),
23778                Duration::from_secs(1),
23779            )],
23780            Arc::new(Semaphore::new(1)),
23781            &ctx,
23782            &mut state,
23783            0,
23784        )
23785        .await;
23786        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23787        assert_eq!(result[0].1.turns, 0);
23788        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23789        assert_eq!(
23790            state.resource_block.unwrap().cause,
23791            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23792        );
23793        assert!(state.active.is_empty());
23794    }
23795
23796    #[tokio::test]
23797    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23798        crate::run::pin_test_home();
23799        let tmp = tempfile::tempdir().unwrap();
23800        let cache = tmp.path().join("target-a");
23801        let guard = hold(&cache);
23802        // Held for longer than the agent's whole 1s budget.
23803        tokio::spawn(async move {
23804            tokio::time::sleep(Duration::from_millis(1500)).await;
23805            guard.release();
23806        });
23807        let mut st = state(60);
23808        let results = run(
23809            vec![job(
23810                sh("alpha", "echo ok"),
23811                tmp.path(),
23812                Duration::from_secs(1),
23813            )],
23814            Arc::new(Semaphore::new(1)),
23815            Some(&cache),
23816            &mut st,
23817        )
23818        .await;
23819        assert!(
23820            matches!(results[0].2, AgentOutcome::Ok(_)),
23821            "the wait must not be charged to the agent"
23822        );
23823        assert!(st.resource_block.is_none());
23824        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23825        assert!(st.resource_wait.is_none() && st.active.is_empty());
23826    }
23827
23828    #[tokio::test]
23829    #[cfg(unix)]
23830    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23831        use std::os::fd::AsRawFd as _;
23832        crate::run::pin_test_home();
23833        let tmp = tempfile::tempdir().unwrap();
23834        crate::git::git(tmp.path(), &["init"]).await.unwrap();
23835        let common = tmp.path().join(".git");
23836        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23837        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23838        assert_eq!(
23839            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23840            0
23841        );
23842        let mut st = state(1);
23843        st.config.coordination = crate::config::test_coordination();
23844        let mut reader = job(
23845            sh("reader", "echo must-not-run"),
23846            tmp.path(),
23847            Duration::from_secs(30),
23848        );
23849        reader.allow_write = false;
23850        let results = run(
23851            vec![reader.clone()],
23852            Arc::new(Semaphore::new(1)),
23853            None,
23854            &mut st,
23855        )
23856        .await;
23857        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23858        assert_eq!(results[0].1.turns, 0);
23859        assert!(FailClass::of(&results[0].2).is_none());
23860        assert_eq!(
23861            st.resource_block.as_ref().unwrap().kind,
23862            "repository coordination"
23863        );
23864        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23865        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23866        drop(external);
23867        let editor = crate::repo_lock::SeatLease::for_checkout(
23868            tmp.path(),
23869            true,
23870            crate::config::test_coordination().as_ref(),
23871        )
23872        .await
23873        .unwrap();
23874        let pause = Pause::new();
23875        let interrupt = Pause::new();
23876        let prompts = Prompts::default();
23877        let ctx = WaveCtx {
23878            run: "r",
23879            node: "review",
23880            prompts: &prompts,
23881            cache: None,
23882            round: None,
23883            carry_seats: false,
23884            pauses: Some((&pause, &interrupt)),
23885        };
23886        let (results, ()) = tokio::join!(
23887            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23888            async {
23889                tokio::time::sleep(Duration::from_millis(50)).await;
23890                pause.park();
23891            }
23892        );
23893        assert!(
23894            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23895        );
23896        assert_eq!(results[0].1.turns, 0);
23897        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23898        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23899        drop(editor);
23900    }
23901
23902    #[tokio::test]
23903    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23904        crate::run::pin_test_home();
23905        let tmp = tempfile::tempdir().unwrap();
23906        let cache = tmp.path().join("target-b");
23907        let _held = hold(&cache);
23908        let mut st = state(1);
23909        let results = run(
23910            vec![job(
23911                sh("alpha", "echo ok"),
23912                tmp.path(),
23913                Duration::from_secs(1),
23914            )],
23915            Arc::new(Semaphore::new(1)),
23916            Some(&cache),
23917            &mut st,
23918        )
23919        .await;
23920        let (_, seat, out) = &results[0];
23921        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23922        assert!(wave_blocked(&results));
23923        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23924        assert_eq!(seat.turns, 0, "nothing was asked");
23925        assert!(
23926            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23927            "no agent was spawned"
23928        );
23929        assert!(FailClass::of(out).is_none(), "not a failure class");
23930        let block = st.resource_block.as_ref().expect("recorded for the resume");
23931        assert_eq!(block.kind, "shared build cache");
23932        assert!(matches!(
23933            block.cause,
23934            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23935        ));
23936        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23937        assert!(st.active.is_empty());
23938    }
23939
23940    #[tokio::test]
23941    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23942        crate::run::pin_test_home();
23943        let tmp = tempfile::tempdir().unwrap();
23944        let mut st = state(1);
23945        let results = run(
23946            vec![job(
23947                sh("alpha", "echo ok"),
23948                tmp.path(),
23949                Duration::from_secs(1),
23950            )],
23951            Arc::new(Semaphore::new(0)),
23952            None,
23953            &mut st,
23954        )
23955        .await;
23956        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23957        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23958        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23959    }
23960
23961    #[tokio::test]
23962    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23963        crate::run::pin_test_home();
23964        let tmp = tempfile::tempdir().unwrap();
23965        let mut st = state(60);
23966        let results = run(
23967            vec![job(
23968                sh("alpha", "sleep 30"),
23969                tmp.path(),
23970                Duration::from_secs(1),
23971            )],
23972            Arc::new(Semaphore::new(1)),
23973            None,
23974            &mut st,
23975        )
23976        .await;
23977        let out = &results[0].2;
23978        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23979        let class = FailClass::of(out).expect("a failure");
23980        assert_eq!(class, FailClass::Timeout);
23981        assert!(should_hand_over(None, &class), "existing handover policy");
23982        assert!(st.resource_block.is_none());
23983    }
23984}